2024-05-20T22:17:09.638 INFO:root:teuthology version: 0.0.1.dev287+gb9e3da8 2024-05-20T22:17:09.638 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --name adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi --archive /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802 --description orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --owner scheduled_adking@teuthology -- /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802/orig.config.yaml 2024-05-20T22:17:09.683 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-20T22:17:09.767 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802 branch: wip-adk4-testing-2024-05-17-0821-squid description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: adking@redhat.com first_in_suite: false job_id: '7715802' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk4-testing-2024-05-17-0821-squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 8a373da3c102b503bf0a94126977cebf498afc8b ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 8a373da3c102b503bf0a94126977cebf498afc8b workunit: branch: wip-adk4-testing-2024-05-17-0821-squid sha1: 8a373da3c102b503bf0a94126977cebf498afc8b owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 1485 sha1: 8a373da3c102b503bf0a94126977cebf498afc8b sleep_before_teardown: 0 subset: 1/18 suite: orch:cephadm suite_branch: wip-adk4-testing-2024-05-17-0821-squid suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_8a373da3c102b503bf0a94126977cebf498afc8b/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 8a373da3c102b503bf0a94126977cebf498afc8b targets: smithi055.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFI330krgTXMmScN6LXWXKS7pWvab5rK6cR/96oi+yoO3/MsxWT+vrqMsvqhLHXvZEZsznjn/+iG2pHi2MSUDNI= smithi138.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBANkW3M2OHKrqYMgIccLE/SZjVYhgVJ6594Q9ahjpxeO0Q1mGmmlx8cbkLSDTgo2xX0/SjoTJQj4ds+hENWuwkY= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: b9e3da87b1ec7920937cb4902b7e3fd9eda86b4a timestamp: 2024-05-20_18:23:59 tube: smithi user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2698841 2024-05-20T22:17:09.767 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_8a373da3c102b503bf0a94126977cebf498afc8b/qa; will attempt to use it 2024-05-20T22:17:09.768 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_8a373da3c102b503bf0a94126977cebf498afc8b/qa/tasks 2024-05-20T22:17:09.768 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-05-20T22:17:09.769 INFO:teuthology.task.internal:Checking packages... 2024-05-20T22:17:09.790 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '8a373da3c102b503bf0a94126977cebf498afc8b' 2024-05-20T22:17:09.790 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-05-20T22:17:09.790 INFO:teuthology.packaging:ref: None 2024-05-20T22:17:09.790 INFO:teuthology.packaging:tag: None 2024-05-20T22:17:09.791 INFO:teuthology.packaging:branch: wip-adk4-testing-2024-05-17-0821-squid 2024-05-20T22:17:09.791 INFO:teuthology.packaging:sha1: 8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:17:09.791 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk4-testing-2024-05-17-0821-squid 2024-05-20T22:17:10.006 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2695.g8a373da3 2024-05-20T22:17:10.008 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-05-20T22:17:10.015 INFO:teuthology.task.internal:no buildpackages task found 2024-05-20T22:17:10.015 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-05-20T22:17:10.057 INFO:teuthology.task.internal:Saving configuration 2024-05-20T22:17:10.075 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-05-20T22:17:10.096 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-20T22:17:10.121 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi055.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-20 22:12:15.902534', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFI330krgTXMmScN6LXWXKS7pWvab5rK6cR/96oi+yoO3/MsxWT+vrqMsvqhLHXvZEZsznjn/+iG2pHi2MSUDNI='} 2024-05-20T22:17:10.145 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi138.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-20 22:12:15.900681', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBANkW3M2OHKrqYMgIccLE/SZjVYhgVJ6594Q9ahjpxeO0Q1mGmmlx8cbkLSDTgo2xX0/SjoTJQj4ds+hENWuwkY='} 2024-05-20T22:17:10.145 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-05-20T22:17:10.195 INFO:teuthology.task.internal:roles: ubuntu@smithi055.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-05-20T22:17:10.195 INFO:teuthology.task.internal:roles: ubuntu@smithi138.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-05-20T22:17:10.195 INFO:teuthology.run_tasks:Running task console_log... 2024-05-20T22:17:10.311 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f112c053ac0>, signals=[15]) 2024-05-20T22:17:10.311 INFO:teuthology.run_tasks:Running task internal.connect... 2024-05-20T22:17:10.352 INFO:teuthology.task.internal:Opening connections... 2024-05-20T22:17:10.352 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi055.front.sepia.ceph.com 2024-05-20T22:17:10.353 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi055.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:17:10.429 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi138.front.sepia.ceph.com 2024-05-20T22:17:10.430 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:17:10.506 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-05-20T22:17:10.512 DEBUG:teuthology.orchestra.run.smithi055:> uname -m 2024-05-20T22:17:10.531 INFO:teuthology.orchestra.run.smithi055.stdout:x86_64 2024-05-20T22:17:10.531 DEBUG:teuthology.orchestra.run.smithi055:> cat /etc/os-release 2024-05-20T22:17:10.589 INFO:teuthology.orchestra.run.smithi055.stdout:NAME="CentOS Stream" 2024-05-20T22:17:10.589 INFO:teuthology.orchestra.run.smithi055.stdout:VERSION="9" 2024-05-20T22:17:10.589 INFO:teuthology.orchestra.run.smithi055.stdout:ID="centos" 2024-05-20T22:17:10.589 INFO:teuthology.orchestra.run.smithi055.stdout:ID_LIKE="rhel fedora" 2024-05-20T22:17:10.589 INFO:teuthology.orchestra.run.smithi055.stdout:VERSION_ID="9" 2024-05-20T22:17:10.590 INFO:teuthology.orchestra.run.smithi055.stdout:PLATFORM_ID="platform:el9" 2024-05-20T22:17:10.590 INFO:teuthology.orchestra.run.smithi055.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-20T22:17:10.590 INFO:teuthology.orchestra.run.smithi055.stdout:ANSI_COLOR="0;31" 2024-05-20T22:17:10.590 INFO:teuthology.orchestra.run.smithi055.stdout:LOGO="fedora-logo-icon" 2024-05-20T22:17:10.590 INFO:teuthology.orchestra.run.smithi055.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-20T22:17:10.590 INFO:teuthology.orchestra.run.smithi055.stdout:HOME_URL="https://centos.org/" 2024-05-20T22:17:10.590 INFO:teuthology.orchestra.run.smithi055.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-20T22:17:10.590 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-20T22:17:10.590 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-20T22:17:10.591 INFO:teuthology.lock.ops:Updating smithi055.front.sepia.ceph.com on lock server 2024-05-20T22:17:10.614 DEBUG:teuthology.orchestra.run.smithi138:> uname -m 2024-05-20T22:17:10.632 INFO:teuthology.orchestra.run.smithi138.stdout:x86_64 2024-05-20T22:17:10.632 DEBUG:teuthology.orchestra.run.smithi138:> cat /etc/os-release 2024-05-20T22:17:10.689 INFO:teuthology.orchestra.run.smithi138.stdout:NAME="CentOS Stream" 2024-05-20T22:17:10.689 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION="9" 2024-05-20T22:17:10.689 INFO:teuthology.orchestra.run.smithi138.stdout:ID="centos" 2024-05-20T22:17:10.689 INFO:teuthology.orchestra.run.smithi138.stdout:ID_LIKE="rhel fedora" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION_ID="9" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:PLATFORM_ID="platform:el9" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:ANSI_COLOR="0;31" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:LOGO="fedora-logo-icon" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:HOME_URL="https://centos.org/" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-20T22:17:10.690 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-20T22:17:10.690 INFO:teuthology.lock.ops:Updating smithi138.front.sepia.ceph.com on lock server 2024-05-20T22:17:10.714 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-05-20T22:17:10.724 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-05-20T22:17:10.800 INFO:teuthology.task.internal:Checking for old test directory... 2024-05-20T22:17:10.800 DEBUG:teuthology.orchestra.run.smithi055:> test '!' -e /home/ubuntu/cephtest 2024-05-20T22:17:10.803 DEBUG:teuthology.orchestra.run.smithi138:> test '!' -e /home/ubuntu/cephtest 2024-05-20T22:17:10.819 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-05-20T22:17:10.855 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-05-20T22:17:10.856 DEBUG:teuthology.orchestra.run.smithi055:> test -z $(ls -A /var/lib/ceph) 2024-05-20T22:17:10.862 DEBUG:teuthology.orchestra.run.smithi138:> test -z $(ls -A /var/lib/ceph) 2024-05-20T22:17:10.933 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-05-20T22:17:11.021 INFO:teuthology.run_tasks:Running task kernel... 2024-05-20T22:17:11.035 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-05-20T22:17:11.035 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-05-20T22:17:11.035 DEBUG:teuthology.orchestra.run.smithi055:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-20T22:17:11.036 DEBUG:teuthology.orchestra.run.smithi138:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-20T22:17:11.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:17:11.053 DEBUG:teuthology.orchestra.run.smithi138:> uname -r 2024-05-20T22:17:11.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:17:11.054 DEBUG:teuthology.orchestra.run.smithi055:> uname -r 2024-05-20T22:17:11.109 INFO:teuthology.orchestra.run.smithi138.stdout:5.14.0-437.el9.x86_64 2024-05-20T22:17:11.110 INFO:teuthology.task.kernel:Running kernel on smithi138: 5.14.0-437.el9.x86_64 2024-05-20T22:17:11.110 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum install -y kernel 2024-05-20T22:17:11.113 INFO:teuthology.orchestra.run.smithi055.stdout:5.14.0-437.el9.x86_64 2024-05-20T22:17:11.113 INFO:teuthology.task.kernel:Running kernel on smithi055: 5.14.0-437.el9.x86_64 2024-05-20T22:17:11.113 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum install -y kernel 2024-05-20T22:17:14.615 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - BaseOS 4.2 MB/s | 8.1 MB 00:01 2024-05-20T22:17:16.001 INFO:teuthology.orchestra.run.smithi138.stdout:CentOS Stream 9 - BaseOS 2.5 MB/s | 8.1 MB 00:03 2024-05-20T22:17:16.499 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-05-20T22:17:17.819 INFO:teuthology.orchestra.run.smithi138.stdout:CentOS Stream 9 - AppStream 25 MB/s | 19 MB 00:00 2024-05-20T22:17:21.908 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - CRB 3.6 MB/s | 6.2 MB 00:01 2024-05-20T22:17:23.003 INFO:teuthology.orchestra.run.smithi138.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.2 MB 00:01 2024-05-20T22:17:23.575 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - Extras packages 34 kB/s | 16 kB 00:00 2024-05-20T22:17:24.403 INFO:teuthology.orchestra.run.smithi138.stdout:CentOS Stream 9 - Extras packages 68 kB/s | 16 kB 00:00 2024-05-20T22:17:25.588 INFO:teuthology.orchestra.run.smithi138.stdout:Extra Packages for Enterprise Linux 20 MB/s | 21 MB 00:01 2024-05-20T22:17:30.746 INFO:teuthology.orchestra.run.smithi138.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-05-20T22:17:32.612 INFO:teuthology.orchestra.run.smithi138.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-20T22:17:32.612 INFO:teuthology.orchestra.run.smithi138.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-20T22:17:32.711 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2024-05-20T22:17:32.730 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2024-05-20T22:17:32.730 INFO:teuthology.orchestra.run.smithi138.stdout: Package Architecture Version Repository Size 2024-05-20T22:17:32.730 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2024-05-20T22:17:32.730 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2024-05-20T22:17:32.730 INFO:teuthology.orchestra.run.smithi138.stdout: kernel x86_64 5.14.0-447.el9 baseos 576 k 2024-05-20T22:17:32.730 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2024-05-20T22:17:32.730 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-core x86_64 5.14.0-447.el9 baseos 16 M 2024-05-20T22:17:32.730 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-modules x86_64 5.14.0-447.el9 baseos 34 M 2024-05-20T22:17:32.731 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-modules-core x86_64 5.14.0-447.el9 baseos 28 M 2024-05-20T22:17:32.731 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:17:32.731 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2024-05-20T22:17:32.731 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2024-05-20T22:17:32.731 INFO:teuthology.orchestra.run.smithi138.stdout:Install 4 Packages 2024-05-20T22:17:32.731 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:17:32.731 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 78 M 2024-05-20T22:17:32.732 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 123 M 2024-05-20T22:17:32.732 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2024-05-20T22:17:33.448 INFO:teuthology.orchestra.run.smithi138.stdout:(1/4): kernel-5.14.0-447.el9.x86_64.rpm 847 kB/s | 576 kB 00:00 2024-05-20T22:17:38.051 INFO:teuthology.orchestra.run.smithi055.stdout:Extra Packages for Enterprise Linux 1.5 MB/s | 21 MB 00:14 2024-05-20T22:17:43.359 INFO:teuthology.orchestra.run.smithi055.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-05-20T22:17:45.299 INFO:teuthology.orchestra.run.smithi055.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-20T22:17:45.299 INFO:teuthology.orchestra.run.smithi055.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-20T22:17:45.396 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-05-20T22:17:45.410 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-05-20T22:17:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: Package Architecture Version Repository Size 2024-05-20T22:17:45.410 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: kernel x86_64 5.14.0-447.el9 baseos 576 k 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: kernel-core x86_64 5.14.0-447.el9 baseos 16 M 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: kernel-modules x86_64 5.14.0-447.el9 baseos 34 M 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: kernel-modules-core x86_64 5.14.0-447.el9 baseos 28 M 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout:Install 4 Packages 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 78 M 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 123 M 2024-05-20T22:17:45.411 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-05-20T22:17:45.583 INFO:teuthology.orchestra.run.smithi055.stdout:[MIRROR] kernel-5.14.0-447.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-447.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T22:17:45.583 INFO:teuthology.orchestra.run.smithi055.stdout:[MIRROR] kernel-core-5.14.0-447.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-core-5.14.0-447.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T22:17:45.584 INFO:teuthology.orchestra.run.smithi055.stdout:[MIRROR] kernel-modules-5.14.0-447.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-5.14.0-447.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T22:17:46.294 INFO:teuthology.orchestra.run.smithi055.stdout:(1/4): kernel-5.14.0-447.el9.x86_64.rpm 774 kB/s | 576 kB 00:00 2024-05-20T22:17:46.741 INFO:teuthology.orchestra.run.smithi138.stdout:(2/4): kernel-core-5.14.0-447.el9.x86_64.rpm 1.1 MB/s | 16 MB 00:13 2024-05-20T22:17:54.619 INFO:teuthology.orchestra.run.smithi055.stdout:(2/4): kernel-core-5.14.0-447.el9.x86_64.rpm 1.7 MB/s | 16 MB 00:09 2024-05-20T22:18:06.695 INFO:teuthology.orchestra.run.smithi138.stdout:(3/4): kernel-modules-core-5.14.0-447.el9.x86_6 864 kB/s | 28 MB 00:33 2024-05-20T22:18:13.910 INFO:teuthology.orchestra.run.smithi138.stdout:(4/4): kernel-modules-5.14.0-447.el9.x86_64.rpm 847 kB/s | 34 MB 00:41 2024-05-20T22:18:13.911 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2024-05-20T22:18:13.911 INFO:teuthology.orchestra.run.smithi138.stdout:Total 1.9 MB/s | 78 MB 00:41 2024-05-20T22:18:14.587 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2024-05-20T22:18:14.942 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2024-05-20T22:18:14.942 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2024-05-20T22:18:15.851 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2024-05-20T22:18:15.852 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2024-05-20T22:18:18.242 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2024-05-20T22:18:18.699 INFO:teuthology.orchestra.run.smithi055.stdout:(3/4): kernel-modules-core-5.14.0-447.el9.x86_6 886 kB/s | 28 MB 00:32 2024-05-20T22:18:18.939 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : kernel-modules-core-5.14.0-447.el9.x86_64 1/4 2024-05-20T22:18:19.037 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:18:21.703 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:18:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:18:26.525 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:18:26.642 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : kernel-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:18:26.750 INFO:teuthology.orchestra.run.smithi055.stdout:(4/4): kernel-modules-5.14.0-447.el9.x86_64.rpm 846 kB/s | 34 MB 00:41 2024-05-20T22:18:26.750 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-05-20T22:18:26.750 INFO:teuthology.orchestra.run.smithi055.stdout:Total 1.9 MB/s | 78 MB 00:41 2024-05-20T22:18:27.458 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-05-20T22:18:27.870 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-05-20T22:18:27.870 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-05-20T22:18:28.831 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-05-20T22:18:28.832 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-05-20T22:18:31.366 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:18:31.487 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-05-20T22:18:32.117 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : kernel-modules-core-5.14.0-447.el9.x86_64 1/4 2024-05-20T22:18:32.199 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:18:35.009 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:18:35.137 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:18:40.210 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:18:40.249 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : kernel-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:18:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:19:00.721 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:19:00.758 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:19:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:19:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : kernel-5.14.0-447.el9.x86_64 1/4 2024-05-20T22:19:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:19:01.125 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:19:01.953 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:19:01.954 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:19:01.954 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2024-05-20T22:19:01.954 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-5.14.0-447.el9.x86_64 2024-05-20T22:19:01.954 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-core-5.14.0-447.el9.x86_64 2024-05-20T22:19:01.954 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-modules-5.14.0-447.el9.x86_64 2024-05-20T22:19:01.954 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-modules-core-5.14.0-447.el9.x86_64 2024-05-20T22:19:01.954 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:19:01.954 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2024-05-20T22:19:02.293 DEBUG:teuthology.orchestra.run.smithi138:> rpm -q kernel | sort -rV | head -n 1 2024-05-20T22:19:02.367 INFO:teuthology.orchestra.run.smithi138.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-20T22:19:02.367 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-20T22:19:02.368 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-447.el9.x86_64 2024-05-20T22:19:02.368 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-20T22:19:02.368 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-20T22:19:02.368 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-05-20T22:19:02.368 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi138.front.sepia.ceph.com, path=None, version=distro) 2024-05-20T22:19:02.368 DEBUG:teuthology.orchestra.run.smithi138:> rpm -q kernel | sort -rV | head -n 1 2024-05-20T22:19:02.417 INFO:teuthology.orchestra.run.smithi138.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-20T22:19:02.418 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-20T22:19:02.418 DEBUG:teuthology.orchestra.run.smithi138:> sudo rpm -qi grub2-tools 2024-05-20T22:19:02.448 INFO:teuthology.orchestra.run.smithi138.stdout:Name : grub2-tools 2024-05-20T22:19:02.449 INFO:teuthology.orchestra.run.smithi138.stdout:Epoch : 1 2024-05-20T22:19:02.449 INFO:teuthology.orchestra.run.smithi138.stdout:Version : 2.06 2024-05-20T22:19:02.449 INFO:teuthology.orchestra.run.smithi138.stdout:Release : 68.el9 2024-05-20T22:19:02.449 INFO:teuthology.orchestra.run.smithi138.stdout:Architecture: x86_64 2024-05-20T22:19:02.449 INFO:teuthology.orchestra.run.smithi138.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-20T22:19:02.449 INFO:teuthology.orchestra.run.smithi138.stdout:Group : Unspecified 2024-05-20T22:19:02.449 INFO:teuthology.orchestra.run.smithi138.stdout:Size : 8264254 2024-05-20T22:19:02.449 INFO:teuthology.orchestra.run.smithi138.stdout:License : GPLv3+ 2024-05-20T22:19:02.449 INFO:teuthology.orchestra.run.smithi138.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:Packager : builder@centos.org 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:Vendor : CentOS 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:Summary : Support tools for GRUB. 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:Description : 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-20T22:19:02.450 INFO:teuthology.orchestra.run.smithi138.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-20T22:19:02.451 INFO:teuthology.orchestra.run.smithi138.stdout:hardware devices. 2024-05-20T22:19:02.451 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:19:02.451 INFO:teuthology.orchestra.run.smithi138.stdout:This subpackage provides tools for support of all platforms. 2024-05-20T22:19:02.451 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-20T22:19:02.452 INFO:teuthology.task.kernel:Updating grub on smithi138 to boot 5.14.0-447.el9.x86_64 2024-05-20T22:19:02.452 DEBUG:teuthology.orchestra.run.smithi138:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-20T22:19:03.102 INFO:teuthology.orchestra.run.smithi138.stderr:Generating grub configuration file ... 2024-05-20T22:19:04.503 INFO:teuthology.orchestra.run.smithi138.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-20T22:19:04.529 INFO:teuthology.orchestra.run.smithi138.stderr:done 2024-05-20T22:19:04.531 DEBUG:teuthology.orchestra.run.smithi138:> mktemp 2024-05-20T22:19:04.547 INFO:teuthology.orchestra.run.smithi138.stdout:/tmp/tmp.6XnHiQ9L3M 2024-05-20T22:19:04.548 DEBUG:teuthology.orchestra.run.smithi138:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.6XnHiQ9L3M 2024-05-20T22:19:04.612 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 0666 /tmp/tmp.6XnHiQ9L3M 2024-05-20T22:19:04.747 DEBUG:teuthology.orchestra.remote:smithi138:/tmp/tmp.6XnHiQ9L3M is 6KB 2024-05-20T22:19:04.766 DEBUG:teuthology.orchestra.run.smithi138:> rm -fr /tmp/tmp.6XnHiQ9L3M 2024-05-20T22:19:04.780 DEBUG:teuthology.orchestra.run.smithi138:> sudo /bin/ls /boot/loader/entries || true 2024-05-20T22:19:04.842 INFO:teuthology.orchestra.run.smithi138.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-20T22:19:04.842 INFO:teuthology.orchestra.run.smithi138.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-20T22:19:04.842 INFO:teuthology.orchestra.run.smithi138.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-20T22:19:04.842 INFO:teuthology.orchestra.run.smithi138.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64.conf 2024-05-20T22:19:04.843 DEBUG:teuthology.orchestra.run.smithi138:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64 2024-05-20T22:19:04.993 DEBUG:teuthology.orchestra.run.smithi138:> sudo shutdown -r now 2024-05-20T22:19:16.554 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:19:16.594 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:19:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: kernel-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:19:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : kernel-5.14.0-447.el9.x86_64 1/4 2024-05-20T22:19:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:19:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:19:17.739 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:19:17.739 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:19:17.739 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-05-20T22:19:17.739 INFO:teuthology.orchestra.run.smithi055.stdout: kernel-5.14.0-447.el9.x86_64 2024-05-20T22:19:17.739 INFO:teuthology.orchestra.run.smithi055.stdout: kernel-core-5.14.0-447.el9.x86_64 2024-05-20T22:19:17.740 INFO:teuthology.orchestra.run.smithi055.stdout: kernel-modules-5.14.0-447.el9.x86_64 2024-05-20T22:19:17.740 INFO:teuthology.orchestra.run.smithi055.stdout: kernel-modules-core-5.14.0-447.el9.x86_64 2024-05-20T22:19:17.740 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:19:17.740 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-05-20T22:19:18.129 DEBUG:teuthology.orchestra.run.smithi055:> rpm -q kernel | sort -rV | head -n 1 2024-05-20T22:19:18.210 INFO:teuthology.orchestra.run.smithi055.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-20T22:19:18.210 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-20T22:19:18.210 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-447.el9.x86_64 2024-05-20T22:19:18.210 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-20T22:19:18.210 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-20T22:19:18.211 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-05-20T22:19:18.211 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi055.front.sepia.ceph.com, path=None, version=distro) 2024-05-20T22:19:18.211 DEBUG:teuthology.orchestra.run.smithi055:> rpm -q kernel | sort -rV | head -n 1 2024-05-20T22:19:18.262 INFO:teuthology.orchestra.run.smithi055.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-20T22:19:18.262 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-20T22:19:18.263 DEBUG:teuthology.orchestra.run.smithi055:> sudo rpm -qi grub2-tools 2024-05-20T22:19:18.306 INFO:teuthology.orchestra.run.smithi055.stdout:Name : grub2-tools 2024-05-20T22:19:18.306 INFO:teuthology.orchestra.run.smithi055.stdout:Epoch : 1 2024-05-20T22:19:18.306 INFO:teuthology.orchestra.run.smithi055.stdout:Version : 2.06 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Release : 68.el9 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Architecture: x86_64 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Group : Unspecified 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Size : 8264254 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:License : GPLv3+ 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Packager : builder@centos.org 2024-05-20T22:19:18.307 INFO:teuthology.orchestra.run.smithi055.stdout:Vendor : CentOS 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout:Summary : Support tools for GRUB. 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout:Description : 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout:hardware devices. 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:19:18.308 INFO:teuthology.orchestra.run.smithi055.stdout:This subpackage provides tools for support of all platforms. 2024-05-20T22:19:18.309 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-20T22:19:18.309 INFO:teuthology.task.kernel:Updating grub on smithi055 to boot 5.14.0-447.el9.x86_64 2024-05-20T22:19:18.309 DEBUG:teuthology.orchestra.run.smithi055:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-20T22:19:18.973 INFO:teuthology.orchestra.run.smithi055.stderr:Generating grub configuration file ... 2024-05-20T22:19:20.398 INFO:teuthology.orchestra.run.smithi055.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-20T22:19:20.435 INFO:teuthology.orchestra.run.smithi055.stderr:done 2024-05-20T22:19:20.437 DEBUG:teuthology.orchestra.run.smithi055:> mktemp 2024-05-20T22:19:20.493 INFO:teuthology.orchestra.run.smithi055.stdout:/tmp/tmp.ntZEgXPKiR 2024-05-20T22:19:20.493 DEBUG:teuthology.orchestra.run.smithi055:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ntZEgXPKiR 2024-05-20T22:19:20.568 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod 0666 /tmp/tmp.ntZEgXPKiR 2024-05-20T22:19:20.709 DEBUG:teuthology.orchestra.remote:smithi055:/tmp/tmp.ntZEgXPKiR is 6KB 2024-05-20T22:19:20.731 DEBUG:teuthology.orchestra.run.smithi055:> rm -fr /tmp/tmp.ntZEgXPKiR 2024-05-20T22:19:20.747 DEBUG:teuthology.orchestra.run.smithi055:> sudo /bin/ls /boot/loader/entries || true 2024-05-20T22:19:20.809 INFO:teuthology.orchestra.run.smithi055.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-20T22:19:20.809 INFO:teuthology.orchestra.run.smithi055.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-20T22:19:20.809 INFO:teuthology.orchestra.run.smithi055.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-20T22:19:20.809 INFO:teuthology.orchestra.run.smithi055.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64.conf 2024-05-20T22:19:20.811 DEBUG:teuthology.orchestra.run.smithi055:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64 2024-05-20T22:19:20.954 DEBUG:teuthology.orchestra.run.smithi055:> sudo shutdown -r now 2024-05-20T22:19:35.013 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-05-20T22:19:35.014 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-05-20T22:19:35.015 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:19:50.970 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-05-20T22:19:50.971 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi055.front.sepia.ceph.com' 2024-05-20T22:19:50.972 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi055.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:19:53.502 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.138 2024-05-20T22:20:02.510 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-05-20T22:20:02.511 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:20:05.566 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.138 2024-05-20T22:20:17.576 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-05-20T22:20:17.577 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:20:49.496 DEBUG:teuthology.orchestra.run.smithi138:> true 2024-05-20T22:20:49.902 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-05-20T22:20:49.902 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-447.el9.x86_64"... 2024-05-20T22:20:49.902 DEBUG:teuthology.orchestra.run.smithi138:> uname -r 2024-05-20T22:20:49.916 INFO:teuthology.orchestra.run.smithi138.stdout:5.14.0-447.el9.x86_64 2024-05-20T22:20:49.917 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-447.el9.x86_64 vs 5.14.0-447.el9.x86_64 2024-05-20T22:20:49.917 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-20T22:20:49.917 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-20T22:20:50.918 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-05-20T22:20:50.918 DEBUG:teuthology.orchestra.run.smithi138:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-20T22:20:50.974 DEBUG:teuthology.orchestra.remote:timed out 2024-05-20T22:20:51.090 INFO:teuthology.orchestra.run.smithi138.stdout:ttyS1 2024-05-20T22:20:51.164 DEBUG:teuthology.parallel:result is None 2024-05-20T22:20:59.975 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi055.front.sepia.ceph.com' 2024-05-20T22:20:59.976 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi055.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:21:00.359 DEBUG:teuthology.orchestra.run.smithi055:> true 2024-05-20T22:21:00.695 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi055.front.sepia.ceph.com' 2024-05-20T22:21:00.696 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-447.el9.x86_64"... 2024-05-20T22:21:00.696 DEBUG:teuthology.orchestra.run.smithi055:> uname -r 2024-05-20T22:21:00.711 INFO:teuthology.orchestra.run.smithi055.stdout:5.14.0-447.el9.x86_64 2024-05-20T22:21:00.711 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-447.el9.x86_64 vs 5.14.0-447.el9.x86_64 2024-05-20T22:21:00.711 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-20T22:21:00.711 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-20T22:21:01.712 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-05-20T22:21:01.712 DEBUG:teuthology.orchestra.run.smithi055:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-20T22:21:01.774 INFO:teuthology.orchestra.run.smithi055.stdout:ttyS1 2024-05-20T22:21:01.805 DEBUG:teuthology.parallel:result is None 2024-05-20T22:21:01.805 INFO:teuthology.run_tasks:Running task internal.base... 2024-05-20T22:21:01.812 INFO:teuthology.task.internal:Creating test directory... 2024-05-20T22:21:01.813 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-20T22:21:01.815 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-20T22:21:01.834 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-05-20T22:21:01.840 INFO:teuthology.run_tasks:Running task internal.archive... 2024-05-20T22:21:01.847 INFO:teuthology.task.internal:Creating archive directory... 2024-05-20T22:21:01.847 DEBUG:teuthology.orchestra.run.smithi055:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-20T22:21:01.873 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-20T22:21:02.144 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-05-20T22:21:02.150 INFO:teuthology.task.internal:Enabling coredump saving... 2024-05-20T22:21:02.150 DEBUG:teuthology.orchestra.run.smithi055:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-20T22:21:02.153 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-20T22:21:02.186 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T22:21:02.190 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T22:21:02.197 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T22:21:02.203 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T22:21:02.205 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-05-20T22:21:02.257 INFO:teuthology.task.internal:Configuring sudo... 2024-05-20T22:21:02.257 DEBUG:teuthology.orchestra.run.smithi055:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-20T22:21:02.260 DEBUG:teuthology.orchestra.run.smithi138:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-20T22:21:02.288 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-05-20T22:21:02.317 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-05-20T22:21:02.317 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-20T22:21:02.329 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-20T22:21:02.346 DEBUG:teuthology.orchestra.run.smithi055:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T22:21:02.428 DEBUG:teuthology.orchestra.run.smithi055:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T22:21:02.483 DEBUG:teuthology.orchestra.run.smithi055:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T22:21:02.568 DEBUG:teuthology.orchestra.run.smithi055:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T22:21:02.609 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:21:02.609 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-20T22:21:02.676 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T22:21:02.720 DEBUG:teuthology.orchestra.run.smithi138:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T22:21:02.771 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T22:21:02.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T22:21:02.896 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-05-20T22:21:02.896 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-20T22:21:02.963 DEBUG:teuthology.orchestra.run.smithi055:> sudo service rsyslog restart 2024-05-20T22:21:02.965 DEBUG:teuthology.orchestra.run.smithi138:> sudo service rsyslog restart 2024-05-20T22:21:03.004 INFO:teuthology.orchestra.run.smithi055.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T22:21:03.051 INFO:teuthology.orchestra.run.smithi138.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T22:21:03.533 INFO:teuthology.run_tasks:Running task internal.timer... 2024-05-20T22:21:03.540 INFO:teuthology.task.internal:Starting timer... 2024-05-20T22:21:03.540 INFO:teuthology.run_tasks:Running task pcp... 2024-05-20T22:21:03.553 INFO:teuthology.run_tasks:Running task selinux... 2024-05-20T22:21:03.561 DEBUG:teuthology.orchestra.run.smithi055:> sudo service auditd rotate 2024-05-20T22:21:03.644 INFO:teuthology.orchestra.run.smithi055.stdout:Rotating logs: 2024-05-20T22:21:03.647 DEBUG:teuthology.orchestra.run.smithi138:> sudo service auditd rotate 2024-05-20T22:21:03.715 INFO:teuthology.orchestra.run.smithi138.stdout:Rotating logs: 2024-05-20T22:21:03.717 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-05-20T22:21:03.717 DEBUG:teuthology.orchestra.run.smithi055:> /usr/sbin/getenforce 2024-05-20T22:21:03.743 INFO:teuthology.orchestra.run.smithi055.stdout:Permissive 2024-05-20T22:21:03.743 DEBUG:teuthology.orchestra.run.smithi138:> /usr/sbin/getenforce 2024-05-20T22:21:03.769 INFO:teuthology.orchestra.run.smithi138.stdout:Permissive 2024-05-20T22:21:03.770 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi055.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi138.front.sepia.ceph.com': 'permissive'} 2024-05-20T22:21:03.770 DEBUG:teuthology.orchestra.run.smithi055:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-20T22:21:03.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:21:03.814 DEBUG:teuthology.orchestra.run.smithi138:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-20T22:21:03.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:21:03.840 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-05-20T22:21:03.840 DEBUG:teuthology.orchestra.run.smithi055:> sudo /usr/sbin/setenforce permissive 2024-05-20T22:21:03.883 DEBUG:teuthology.orchestra.run.smithi138:> sudo /usr/sbin/setenforce permissive 2024-05-20T22:21:03.909 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-05-20T22:21:03.919 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-05-20T22:21:03.930 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-05-20T22:21:03.931 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi055.front.sepia.ceph.com,smithi138.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-05-20T22:25:28.670 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi055.front.sepia.ceph.com'), Remote(name='ubuntu@smithi138.front.sepia.ceph.com')] 2024-05-20T22:25:28.671 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi055.front.sepia.ceph.com' 2024-05-20T22:25:28.672 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi055.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:25:28.754 DEBUG:teuthology.orchestra.run.smithi055:> true 2024-05-20T22:25:28.844 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi055.front.sepia.ceph.com' 2024-05-20T22:25:28.844 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-05-20T22:25:28.845 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:25:28.922 DEBUG:teuthology.orchestra.run.smithi138:> true 2024-05-20T22:25:29.001 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-05-20T22:25:29.002 INFO:teuthology.run_tasks:Running task clock... 2024-05-20T22:25:29.012 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-05-20T22:25:29.012 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-20T22:25:29.012 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T22:25:29.015 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-20T22:25:29.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T22:25:29.046 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-20T22:25:29.062 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-20T22:25:29.072 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-20T22:25:29.086 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-20T22:25:29.088 INFO:teuthology.orchestra.run.smithi055.stderr:sudo: ntpd: command not found 2024-05-20T22:25:29.101 INFO:teuthology.orchestra.run.smithi055.stdout:506 Cannot talk to daemon 2024-05-20T22:25:29.110 INFO:teuthology.orchestra.run.smithi138.stderr:sudo: ntpd: command not found 2024-05-20T22:25:29.115 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-20T22:25:29.121 INFO:teuthology.orchestra.run.smithi138.stdout:506 Cannot talk to daemon 2024-05-20T22:25:29.130 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-20T22:25:29.135 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-20T22:25:29.149 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-20T22:25:29.189 INFO:teuthology.orchestra.run.smithi055.stderr:bash: line 1: ntpq: command not found 2024-05-20T22:25:29.193 INFO:teuthology.orchestra.run.smithi055.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T22:25:29.193 INFO:teuthology.orchestra.run.smithi055.stdout:=============================================================================== 2024-05-20T22:25:29.193 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:25:29.193 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:25:29.193 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:25:29.193 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:25:29.202 INFO:teuthology.orchestra.run.smithi138.stderr:bash: line 1: ntpq: command not found 2024-05-20T22:25:29.206 INFO:teuthology.orchestra.run.smithi138.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T22:25:29.207 INFO:teuthology.orchestra.run.smithi138.stdout:=============================================================================== 2024-05-20T22:25:29.207 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:25:29.207 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:25:29.207 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:25:29.207 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:25:29.207 INFO:teuthology.run_tasks:Running task cephadm... 2024-05-20T22:25:29.303 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '8a373da3c102b503bf0a94126977cebf498afc8b'} 2024-05-20T22:25:29.303 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-05-20T22:25:29.303 INFO:tasks.cephadm:Cluster fsid is dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:25:29.303 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-05-20T22:25:29.304 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.55', 'mon.c': '[v2:172.21.15.55:3301,v1:172.21.15.55:6790]', 'mon.b': '172.21.15.138'} 2024-05-20T22:25:29.304 INFO:tasks.cephadm:First mon is mon.a on smithi055 2024-05-20T22:25:29.304 INFO:tasks.cephadm:First mgr is y 2024-05-20T22:25:29.304 INFO:tasks.cephadm:Normalizing hostnames... 2024-05-20T22:25:29.304 DEBUG:teuthology.orchestra.run.smithi055:> sudo hostname $(hostname -s) 2024-05-20T22:25:29.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo hostname $(hostname -s) 2024-05-20T22:25:29.357 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-05-20T22:25:29.357 DEBUG:teuthology.orchestra.run.smithi055:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-20T22:25:29.811 INFO:teuthology.orchestra.run.smithi055.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 20 22:25 /home/ubuntu/cephtest/cephadm 2024-05-20T22:25:29.812 DEBUG:teuthology.orchestra.run.smithi138:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-20T22:25:30.143 INFO:teuthology.orchestra.run.smithi138.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 20 22:25 /home/ubuntu/cephtest/cephadm 2024-05-20T22:25:30.143 DEBUG:teuthology.orchestra.run.smithi055:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-20T22:25:30.163 DEBUG:teuthology.orchestra.run.smithi138:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-20T22:25:30.189 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-05-20T22:25:30.190 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-20T22:25:30.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-20T22:25:31.115 INFO:teuthology.orchestra.run.smithi138.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-20T22:25:31.271 INFO:teuthology.orchestra.run.smithi055.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-20T22:25:51.716 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:25:51.716 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-20T22:25:51.717 INFO:teuthology.orchestra.run.smithi055.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-20T22:25:51.717 INFO:teuthology.orchestra.run.smithi055.stdout: "repo_digests": [ 2024-05-20T22:25:51.717 INFO:teuthology.orchestra.run.smithi055.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-20T22:25:51.717 INFO:teuthology.orchestra.run.smithi055.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-20T22:25:51.717 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2024-05-20T22:25:51.717 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:25:52.250 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2024-05-20T22:25:52.250 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-20T22:25:52.250 INFO:teuthology.orchestra.run.smithi138.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-20T22:25:52.250 INFO:teuthology.orchestra.run.smithi138.stdout: "repo_digests": [ 2024-05-20T22:25:52.250 INFO:teuthology.orchestra.run.smithi138.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-20T22:25:52.250 INFO:teuthology.orchestra.run.smithi138.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-20T22:25:52.251 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2024-05-20T22:25:52.251 INFO:teuthology.orchestra.run.smithi138.stdout:} 2024-05-20T22:25:52.269 DEBUG:teuthology.orchestra.run.smithi055:> sudo mkdir -p /etc/ceph 2024-05-20T22:25:52.298 DEBUG:teuthology.orchestra.run.smithi138:> sudo mkdir -p /etc/ceph 2024-05-20T22:25:52.331 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod 777 /etc/ceph 2024-05-20T22:25:52.363 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 777 /etc/ceph 2024-05-20T22:25:52.395 INFO:tasks.cephadm:Writing seed config... 2024-05-20T22:25:52.396 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-05-20T22:25:52.396 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-05-20T22:25:52.396 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-05-20T22:25:52.397 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-05-20T22:25:52.397 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-05-20T22:25:52.397 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-05-20T22:25:52.397 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-05-20T22:25:52.397 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-05-20T22:25:52.397 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-05-20T22:25:52.397 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:25:52.398 DEBUG:teuthology.orchestra.run.smithi055:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-05-20T22:25:52.420 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = dd0bace8-16f7-11ef-bc9a-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-05-20T22:25:52.420 DEBUG:teuthology.orchestra.run.smithi055:mon.a> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.a.service 2024-05-20T22:25:52.462 DEBUG:teuthology.orchestra.run.smithi055:mgr.y> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.y.service 2024-05-20T22:25:52.505 INFO:tasks.cephadm:Bootstrapping... 2024-05-20T22:25:52.505 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.55 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:25:52.706 INFO:teuthology.orchestra.run.smithi055.stderr:-------------------------------------------------------------------------------- 2024-05-20T22:25:52.706 INFO:teuthology.orchestra.run.smithi055.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'dd0bace8-16f7-11ef-bc9a-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.55', '--skip-admin-label'] 2024-05-20T22:25:52.730 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: 5.0.2 2024-05-20T22:25:52.733 INFO:teuthology.orchestra.run.smithi055.stderr:Verifying podman|docker is present... 2024-05-20T22:25:52.755 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: 5.0.2 2024-05-20T22:25:52.757 INFO:teuthology.orchestra.run.smithi055.stderr:Verifying lvm2 is present... 2024-05-20T22:25:52.757 INFO:teuthology.orchestra.run.smithi055.stderr:Verifying time synchronization is in place... 2024-05-20T22:25:52.765 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-20T22:25:52.771 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: inactive 2024-05-20T22:25:52.779 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: enabled 2024-05-20T22:25:52.785 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: active 2024-05-20T22:25:52.786 INFO:teuthology.orchestra.run.smithi055.stderr:Unit chronyd.service is enabled and running 2024-05-20T22:25:52.786 INFO:teuthology.orchestra.run.smithi055.stderr:Repeating the final host check... 2024-05-20T22:25:52.807 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: 5.0.2 2024-05-20T22:25:52.809 INFO:teuthology.orchestra.run.smithi055.stderr:podman (/bin/podman) version 5.0.2 is present 2024-05-20T22:25:52.810 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl is present 2024-05-20T22:25:52.810 INFO:teuthology.orchestra.run.smithi055.stderr:lvcreate is present 2024-05-20T22:25:52.816 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-20T22:25:52.823 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: inactive 2024-05-20T22:25:52.830 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: enabled 2024-05-20T22:25:52.836 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: active 2024-05-20T22:25:52.837 INFO:teuthology.orchestra.run.smithi055.stderr:Unit chronyd.service is enabled and running 2024-05-20T22:25:52.837 INFO:teuthology.orchestra.run.smithi055.stderr:Host looks OK 2024-05-20T22:25:52.837 INFO:teuthology.orchestra.run.smithi055.stderr:Cluster fsid: dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:25:52.838 INFO:teuthology.orchestra.run.smithi055.stderr:Acquiring lock 140583005090816 on /run/cephadm/dd0bace8-16f7-11ef-bc9a-c7b262605968.lock 2024-05-20T22:25:52.838 INFO:teuthology.orchestra.run.smithi055.stderr:Lock 140583005090816 acquired on /run/cephadm/dd0bace8-16f7-11ef-bc9a-c7b262605968.lock 2024-05-20T22:25:52.838 INFO:teuthology.orchestra.run.smithi055.stderr:Verifying IP 172.21.15.55 port 3300 ... 2024-05-20T22:25:52.838 INFO:teuthology.orchestra.run.smithi055.stderr:Verifying IP 172.21.15.55 port 6789 ... 2024-05-20T22:25:52.838 INFO:teuthology.orchestra.run.smithi055.stderr:Base mon IP is 172.21.15.55, final addrv is [v2:172.21.15.55:3300,v1:172.21.15.55:6789] 2024-05-20T22:25:52.841 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.55 metric 100 2024-05-20T22:25:52.841 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.55 metric 100 2024-05-20T22:25:52.844 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-05-20T22:25:52.844 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-05-20T22:25:52.844 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-05-20T22:25:52.846 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-05-20T22:25:52.847 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-05-20T22:25:52.847 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-20T22:25:52.847 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-05-20T22:25:52.847 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fef8:2606/64 scope link noprefixroute 2024-05-20T22:25:52.847 INFO:teuthology.orchestra.run.smithi055.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-20T22:25:52.847 INFO:teuthology.orchestra.run.smithi055.stderr:Mon IP `172.21.15.55` is in CIDR network `172.21.0.0/20` 2024-05-20T22:25:52.847 INFO:teuthology.orchestra.run.smithi055.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-05-20T22:25:52.848 INFO:teuthology.orchestra.run.smithi055.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-20T22:25:52.953 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-05-20T22:25:53.301 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: Getting image source signatures 2024-05-20T22:25:53.301 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-05-20T22:25:53.301 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-05-20T22:25:53.301 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-05-20T22:25:53.301 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-05-20T22:25:53.301 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-05-20T22:25:53.301 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-20T22:25:53.454 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: Writing manifest to image destination 2024-05-20T22:25:53.616 INFO:teuthology.orchestra.run.smithi055.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-20T22:25:54.008 INFO:teuthology.orchestra.run.smithi055.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-20T22:25:54.482 INFO:teuthology.orchestra.run.smithi055.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-20T22:25:54.482 INFO:teuthology.orchestra.run.smithi055.stderr:Extracting ceph user uid/gid from container image... 2024-05-20T22:25:54.866 INFO:teuthology.orchestra.run.smithi055.stderr:stat: 167 167 2024-05-20T22:25:55.416 INFO:teuthology.orchestra.run.smithi055.stderr:Creating initial keys... 2024-05-20T22:25:55.839 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph-authtool: AQDzzUtmVt4yLhAAPRKYUs6ZwrBFhltKLIct2w== 2024-05-20T22:25:56.721 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph-authtool: AQD0zUtmWt2jJhAAuBgpA5a0uzUMtfP3c2eH7A== 2024-05-20T22:25:59.039 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph-authtool: AQD2zUtmx9qKDxAAF2atv/bD2kIV6QIUSFLNNg== 2024-05-20T22:26:04.500 INFO:teuthology.orchestra.run.smithi055.stderr:Creating initial monmap... 2024-05-20T22:26:08.074 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-20T22:26:08.075 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-05-20T22:26:08.075 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:26:08.075 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-20T22:26:09.255 INFO:teuthology.orchestra.run.smithi055.stderr:monmaptool for a [v2:172.21.15.55:3300,v1:172.21.15.55:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-20T22:26:09.255 INFO:teuthology.orchestra.run.smithi055.stderr:setting min_mon_release = octopus 2024-05-20T22:26:09.255 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/monmaptool: set fsid to dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:26:09.255 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-20T22:26:09.255 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T22:26:09.255 INFO:teuthology.orchestra.run.smithi055.stderr:Creating mon... 2024-05-20T22:26:10.495 INFO:teuthology.orchestra.run.smithi055.stderr:create mon.a on 2024-05-20T22:26:10.701 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-05-20T22:26:10.890 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968.target → /etc/systemd/system/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968.target. 2024-05-20T22:26:10.890 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968.target → /etc/systemd/system/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968.target. 2024-05-20T22:26:11.223 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: Failed to reset failed state of unit ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.a.service: Unit ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.a.service not loaded. 2024-05-20T22:26:11.231 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: Created symlink /etc/systemd/system/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968.target.wants/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.a.service → /etc/systemd/system/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@.service. 2024-05-20T22:26:12.727 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: disabled 2024-05-20T22:26:12.734 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: inactive 2024-05-20T22:26:12.735 INFO:teuthology.orchestra.run.smithi055.stderr:firewalld.service is not enabled 2024-05-20T22:26:12.735 INFO:teuthology.orchestra.run.smithi055.stderr:Not possible to enable service . firewalld.service is not available 2024-05-20T22:26:12.736 INFO:teuthology.orchestra.run.smithi055.stderr:Waiting for mon to start... 2024-05-20T22:26:12.736 INFO:teuthology.orchestra.run.smithi055.stderr:Waiting for mon... 2024-05-20T22:26:12.863 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:12 smithi055 systemd[1]: Started Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:26:13.293 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: cluster: 2024-05-20T22:26:13.293 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: id: dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:26:13.293 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-05-20T22:26:13.293 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:13.293 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: services: 2024-05-20T22:26:13.293 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.629136s) 2024-05-20T22:26:13.293 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mgr: no daemons active 2024-05-20T22:26:13.294 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-05-20T22:26:13.294 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:13.294 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: data: 2024-05-20T22:26:13.294 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-05-20T22:26:13.294 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-05-20T22:26:13.294 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-05-20T22:26:13.294 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: pgs: 2024-05-20T22:26:13.294 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:13.797 INFO:teuthology.orchestra.run.smithi055.stderr:mon is available 2024-05-20T22:26:13.797 INFO:teuthology.orchestra.run.smithi055.stderr:Assimilating anything we can from ceph.conf... 2024-05-20T22:26:13.903 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:13 smithi055 ceph-mon[24416]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-20T22:26:13.903 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:13 smithi055 ceph-mon[24416]: monmap e1: 1 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0]} 2024-05-20T22:26:13.903 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:13 smithi055 ceph-mon[24416]: fsmap 2024-05-20T22:26:13.903 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:13 smithi055 ceph-mon[24416]: osdmap e1: 0 total, 0 up, 0 in 2024-05-20T22:26:13.903 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:13 smithi055 ceph-mon[24416]: mgrmap e1: no daemons active 2024-05-20T22:26:13.903 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:13 smithi055 ceph-mon[24416]: from='client.? 172.21.15.55:0/2303478710' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:26:14.303 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:14.303 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: [global] 2024-05-20T22:26:14.303 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: fsid = dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:26:14.303 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.55:3300,v1:172.21.15.55:6789] 2024-05-20T22:26:14.303 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-20T22:26:14.303 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-20T22:26:14.303 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-20T22:26:14.303 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-20T22:26:14.304 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:14.304 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: [mgr] 2024-05-20T22:26:14.304 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-05-20T22:26:14.304 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-20T22:26:14.304 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:14.304 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: [osd] 2024-05-20T22:26:14.304 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-20T22:26:14.304 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-20T22:26:14.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:14 smithi055 ceph-mon[24416]: from='client.? 172.21.15.55:0/504722054' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-20T22:26:14.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:14 smithi055 ceph-mon[24416]: from='client.? 172.21.15.55:0/504722054' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-20T22:26:14.820 INFO:teuthology.orchestra.run.smithi055.stderr:Generating new minimal ceph.conf... 2024-05-20T22:26:15.802 INFO:teuthology.orchestra.run.smithi055.stderr:Restarting the monitor... 2024-05-20T22:26:15.848 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:15 smithi055 ceph-mon[24416]: from='client.? 172.21.15.55:0/4244311108' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:15.848 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:15 smithi055 systemd[1]: Stopping Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:26:16.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:15 smithi055 bash[24636]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon.a" found: no such container 2024-05-20T22:26:16.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:15 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a[24411]: 2024-05-20T22:26:15.980+0000 7f2a63a46700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:26:16.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:15 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a[24411]: 2024-05-20T22:26:15.980+0000 7f2a63a46700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-05-20T22:26:16.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 podman[24646]: 2024-05-20 22:26:16.011296664 +0000 UTC m=+0.149933462 container died a5cc4fdd2f4db39389e874afa24ff54a949203dd705850a11e6133272e349636 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, architecture=x86_64, release=754, vcs-type=git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, name=centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-20T22:26:16.602 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 podman[24646]: 2024-05-20 22:26:16.354614649 +0000 UTC m=+0.493251492 container cleanup a5cc4fdd2f4db39389e874afa24ff54a949203dd705850a11e6133272e349636 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-20T22:26:16.602 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 bash[24646]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a 2024-05-20T22:26:16.602 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 bash[24673]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon.a" found: no such container 2024-05-20T22:26:16.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 podman[24659]: 2024-05-20 22:26:16.605520726 +0000 UTC m=+0.594789382 container remove a5cc4fdd2f4db39389e874afa24ff54a949203dd705850a11e6133272e349636 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, version=8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., ceph=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, distribution-scope=public, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.buildah.version=1.19.8, release=754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-20T22:26:16.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.a.service: Deactivated successfully. 2024-05-20T22:26:16.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 systemd[1]: Stopped Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:26:16.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 systemd[1]: Starting Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:26:16.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 podman[24739]: 2024-05-20 22:26:16.75222053 +0000 UTC m=+0.018850107 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-20T22:26:16.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:16 smithi055 podman[24739]: 2024-05-20 22:26:16.864723054 +0000 UTC m=+0.131352619 container create 2a879cf14270394e5475855fbfcb9d92158b49fd7b796de5db1d7ae6c79cc557 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, ceph=True, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vcs-type=git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD) 2024-05-20T22:26:17.114 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 podman[24739]: 2024-05-20 22:26:17.058909794 +0000 UTC m=+0.325539369 container init 2a879cf14270394e5475855fbfcb9d92158b49fd7b796de5db1d7ae6c79cc557 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, version=8, release=754, vcs-type=git, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream) 2024-05-20T22:26:17.114 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 podman[24739]: 2024-05-20 22:26:17.062290892 +0000 UTC m=+0.328920464 container start 2a879cf14270394e5475855fbfcb9d92158b49fd7b796de5db1d7ae6c79cc557 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, CEPH_POINT_RELEASE=-17.2.0, version=8, release=754, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.component=centos-stream-container, architecture=x86_64, vcs-type=git, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-20T22:26:17.114 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T22:26:17.114 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-20T22:26:17.114 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: pidfile_write: ignore empty --pid-file 2024-05-20T22:26:17.114 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: load: jerasure load: lrc 2024-05-20T22:26:17.114 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: RocksDB version: 6.15.5 2024-05-20T22:26:17.114 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Compile date Apr 18 2022 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: DB SUMMARY 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: DB Session ID: 9QJ1WM54EWGHA2M0JHGE 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: CURRENT file: CURRENT 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82627 ; 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.error_if_exists: 0 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.create_if_missing: 0 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.paranoid_checks: 1 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T22:26:17.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.env: 0x5562f52c6860 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.fs: Posix File System 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.info_log: 0x5562f5c51dc0 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.statistics: (nil) 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.use_fsync: 0 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_log_file_size: 0 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.allow_fallocate: 1 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T22:26:17.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.use_direct_reads: 0 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.db_log_dir: 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T22:26:17.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.write_buffer_manager: 0x5562f5d422a0 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T22:26:17.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.unordered_write: 0 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.row_cache: None 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.wal_filter: None 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.preserve_deletes: 0 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.two_write_queues: 0 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.atomic_flush: 0 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T22:26:17.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.log_readahead_size: 0 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_background_jobs: 2 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_background_compactions: -1 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_subcompactions: 1 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T22:26:17.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_open_files: -1 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_background_flushes: -1 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Compression algorithms supported: 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T22:26:17.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: kZSTD supported: 0 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: kXpressCompression supported: 0 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: kLZ4Compression supported: 1 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: kBZip2Compression supported: 0 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: kZlibCompression supported: 1 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: kSnappyCompression supported: 1 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.merge_operator: 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_filter: None 2024-05-20T22:26:17.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5562f5c1dd00) 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: cache_index_and_filter_blocks: 1 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: pin_top_level_index_and_filter: 1 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_type: 0 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: data_block_index_type: 0 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_shortening: 1 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: hash_index_allow_collision: 1 2024-05-20T22:26:17.124 INFO:journalctl@ceph.mon.a.smithi055.stdout: checksum: 1 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: no_block_cache: 0 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache: 0x5562f5c88170 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_name: BinnedLRUCache 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_options: 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: capacity : 536870912 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: num_shard_bits : 4 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: strict_capacity_limit : 0 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: high_pri_pool_ratio: 0.000 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_compressed: (nil) 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: persistent_cache: (nil) 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_size: 4096 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_size_deviation: 10 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_restart_interval: 16 2024-05-20T22:26:17.125 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_block_restart_interval: 1 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: metadata_block_size: 4096 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: partition_filters: 0 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_delta_encoding: 1 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: whole_key_filtering: 1 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: verify_compression: 0 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: read_amp_bytes_per_bit: 0 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: format_version: 4 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: enable_index_compression: 1 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_align: 0 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T22:26:17.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compression: NoCompression 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.num_levels: 7 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T22:26:17.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T22:26:17.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.arena_block_size: 4194304 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T22:26:17.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.table_properties_collectors: 2024-05-20T22:26:17.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.inplace_update_support: 0 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.bloom_locality: 0 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.max_successive_merges: 0 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.ttl: 2592000 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.enable_blob_files: false 2024-05-20T22:26:17.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.min_blob_size: 0 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243977089595, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243977090449, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79177, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77440, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71749, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1716243977, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "193ad7b1-5da5-4698-94d6-09a93e6f82dc", "db_session_id": "9QJ1WM54EWGHA2M0JHGE"}} 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243977091133, "job": 1, "event": "recovery_finished"} 2024-05-20T22:26:17.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5562f5c6ea80 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: DB pointer 0x5562f5c7e000 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** DB Stats ** 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:26:17.133 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.1 0.00 0.00 1 0.001 0 0 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.1 0.00 0.00 1 0.001 0 0 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.1 0.00 0.00 1 0.001 0 0 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 135.1 0.00 0.00 1 0.001 0 0 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:26:17.134 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative compaction: 0.00 GB write, 18.96 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval compaction: 0.00 GB write, 18.96 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.1 0.00 0.00 1 0.001 0 0 2024-05-20T22:26:17.135 INFO:journalctl@ceph.mon.a.smithi055.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 135.1 0.00 0.00 1 0.001 0 0 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 135.1 0.00 0.00 1 0.001 0 0 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative compaction: 0.00 GB write, 18.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:26:17.136 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: starting mon.a rank 0 at public addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] at bind addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mon.a@-1(???) e1 preinit fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mon.a@-1(???).mds e1 new map 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mon.a@-1(???).mds e1 print_map 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout: e1 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout: legacy client fscid: -1 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout: No filesystems configured 2024-05-20T22:26:17.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-20T22:26:17.138 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:26:17.138 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:26:17.138 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:26:17.138 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-05-20T22:26:17.138 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 bash[24739]: 2a879cf14270394e5475855fbfcb9d92158b49fd7b796de5db1d7ae6c79cc557 2024-05-20T22:26:17.138 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 systemd[1]: Started Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:26:17.138 INFO:teuthology.orchestra.run.smithi055.stderr:Setting mon public_network to 172.21.0.0/20 2024-05-20T22:26:17.388 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-20T22:26:17.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: monmap e1: 1 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0]} 2024-05-20T22:26:17.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: fsmap 2024-05-20T22:26:17.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: osdmap e1: 0 total, 0 up, 0 in 2024-05-20T22:26:17.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:17 smithi055 ceph-mon[24757]: mgrmap e1: no daemons active 2024-05-20T22:26:18.166 INFO:teuthology.orchestra.run.smithi055.stderr:Wrote config to /etc/ceph/ceph.conf 2024-05-20T22:26:18.166 INFO:teuthology.orchestra.run.smithi055.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:26:18.166 INFO:teuthology.orchestra.run.smithi055.stderr:Creating mgr... 2024-05-20T22:26:18.166 INFO:teuthology.orchestra.run.smithi055.stderr:Verifying port 9283 ... 2024-05-20T22:26:18.360 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: Failed to reset failed state of unit ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.y.service: Unit ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.y.service not loaded. 2024-05-20T22:26:18.367 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: Created symlink /etc/systemd/system/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968.target.wants/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@.service. 2024-05-20T22:26:18.665 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:18 smithi055 systemd[1]: Starting Ceph mgr.y for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:26:18.940 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:18 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4218079047' entity='client.admin' 2024-05-20T22:26:18.941 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:18 smithi055 podman[24974]: 2024-05-20 22:26:18.757951513 +0000 UTC m=+0.105502843 container create 70d348a53fb955a3abed20c7ca388d7651966e06565a4e0b84ee14c02c9cb8c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vcs-type=git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream) 2024-05-20T22:26:18.941 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:18 smithi055 podman[24974]: 2024-05-20 22:26:18.670842951 +0000 UTC m=+0.018394312 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-20T22:26:18.941 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:18 smithi055 podman[24974]: 2024-05-20 22:26:18.944058927 +0000 UTC m=+0.291610283 container init 70d348a53fb955a3abed20c7ca388d7651966e06565a4e0b84ee14c02c9cb8c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, ceph=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, release=754, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Guillaume Abrioux , version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8) 2024-05-20T22:26:19.012 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: disabled 2024-05-20T22:26:19.020 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: inactive 2024-05-20T22:26:19.020 INFO:teuthology.orchestra.run.smithi055.stderr:firewalld.service is not enabled 2024-05-20T22:26:19.020 INFO:teuthology.orchestra.run.smithi055.stderr:Not possible to enable service . firewalld.service is not available 2024-05-20T22:26:19.029 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: disabled 2024-05-20T22:26:19.037 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: inactive 2024-05-20T22:26:19.037 INFO:teuthology.orchestra.run.smithi055.stderr:firewalld.service is not enabled 2024-05-20T22:26:19.037 INFO:teuthology.orchestra.run.smithi055.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-05-20T22:26:19.037 INFO:teuthology.orchestra.run.smithi055.stderr:Waiting for mgr to start... 2024-05-20T22:26:19.038 INFO:teuthology.orchestra.run.smithi055.stderr:Waiting for mgr... 2024-05-20T22:26:19.231 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:18 smithi055 podman[24974]: 2024-05-20 22:26:18.947358196 +0000 UTC m=+0.294909536 container start 70d348a53fb955a3abed20c7ca388d7651966e06565a4e0b84ee14c02c9cb8c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_CLEAN=True, release=754, name=centos-stream, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.buildah.version=1.19.8, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux ) 2024-05-20T22:26:19.231 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:19 smithi055 bash[24974]: 70d348a53fb955a3abed20c7ca388d7651966e06565a4e0b84ee14c02c9cb8c1 2024-05-20T22:26:19.231 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:19 smithi055 systemd[1]: Started Ceph mgr.y for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:26:19.231 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:19.098+0000 7f3124636000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:26:19.231 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:19.233+0000 7f3124636000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:26:19.488 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:19.338+0000 7f3124636000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:26:19.552 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:19.552 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: { 2024-05-20T22:26:19.552 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "fsid": "dd0bace8-16f7-11ef-bc9a-c7b262605968", 2024-05-20T22:26:19.552 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "health": { 2024-05-20T22:26:19.552 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-20T22:26:19.552 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "checks": {}, 2024-05-20T22:26:19.552 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mutes": [] 2024-05-20T22:26:19.552 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum": [ 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 0 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "a" 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "monmap": { 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:19.553 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osdmap": { 2024-05-20T22:26:19.554 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:19.554 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-20T22:26:19.554 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-20T22:26:19.554 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-20T22:26:19.554 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-20T22:26:19.557 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-20T22:26:19.557 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-20T22:26:19.557 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:19.557 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "pgmap": { 2024-05-20T22:26:19.557 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-20T22:26:19.557 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-20T22:26:19.557 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "fsmap": { 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-20T22:26:19.558 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "available": false, 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "modules": [ 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "iostat", 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "nfs", 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "restful" 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "servicemap": { 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "modified": "2024-05-20T22:26:12.669349+0000", 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:19.559 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-20T22:26:19.560 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: } 2024-05-20T22:26:19.896 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:19 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2896494486' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T22:26:19.897 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:19.669+0000 7f3124636000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:26:19.897 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:19.754+0000 7f3124636000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:26:20.635 INFO:teuthology.orchestra.run.smithi055.stderr:mgr not available, waiting (1/15)... 2024-05-20T22:26:21.207 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:20 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:20.956+0000 7f3124636000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:26:21.207 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:21.125+0000 7f3124636000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:26:21.207 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:21.210+0000 7f3124636000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:26:21.978 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:21.533+0000 7f3124636000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:26:21.979 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:21.626+0000 7f3124636000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:26:21.979 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:21.723+0000 7f3124636000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:26:22.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:21.981+0000 7f3124636000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:26:22.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:22.445+0000 7f3124636000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:26:23.061 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:22.965+0000 7f3124636000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:26:23.186 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:23.186 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: { 2024-05-20T22:26:23.186 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "fsid": "dd0bace8-16f7-11ef-bc9a-c7b262605968", 2024-05-20T22:26:23.186 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "health": { 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "checks": {}, 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mutes": [] 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum": [ 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 0 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "a" 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "monmap": { 2024-05-20T22:26:23.187 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osdmap": { 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:23.188 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "pgmap": { 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "fsmap": { 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-20T22:26:23.189 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "available": false, 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "modules": [ 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "iostat", 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "nfs", 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "restful" 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "servicemap": { 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:23.190 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "modified": "2024-05-20T22:26:12.669349+0000", 2024-05-20T22:26:23.191 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:26:23.191 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:23.191 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-20T22:26:23.191 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: } 2024-05-20T22:26:23.381 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:23 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3821662653' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T22:26:23.381 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:23.063+0000 7f3124636000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:26:23.381 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:23.383+0000 7f3124636000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:26:23.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:23.476+0000 7f3124636000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:26:23.756 INFO:teuthology.orchestra.run.smithi055.stderr:mgr not available, waiting (2/15)... 2024-05-20T22:26:24.028 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:23.894+0000 7f3124636000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:26:24.028 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:24.031+0000 7f3124636000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:26:24.334 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:24.126+0000 7f3124636000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:26:24.335 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:24.244+0000 7f3124636000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:26:24.335 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:24.336+0000 7f3124636000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:26:25.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:24.905+0000 7f3124636000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:26:25.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:25 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:25.283+0000 7f3124636000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:26:25.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: Activating manager daemon y 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: mgrmap e2: y(active, starting, since 0.005136s) 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: Manager daemon y is now available 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' 2024-05-20T22:26:25.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:25 smithi055 ceph-mon[24757]: from='mgr.14100 172.21.15.55:0/977711534' entity='mgr.y' 2024-05-20T22:26:26.268 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: { 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "fsid": "dd0bace8-16f7-11ef-bc9a-c7b262605968", 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "health": { 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "checks": {}, 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mutes": [] 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum": [ 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 0 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:26.269 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "a" 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "monmap": { 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osdmap": { 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-20T22:26:26.270 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "pgmap": { 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-20T22:26:26.274 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "fsmap": { 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "available": false, 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "modules": [ 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "iostat", 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "nfs", 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "restful" 2024-05-20T22:26:26.275 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:26.276 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:26:26.276 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:26.276 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "servicemap": { 2024-05-20T22:26:26.276 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:26.276 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "modified": "2024-05-20T22:26:12.669349+0000", 2024-05-20T22:26:26.276 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:26:26.276 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:26.276 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-20T22:26:26.276 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: } 2024-05-20T22:26:26.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:26 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2618800765' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T22:26:26.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:26 smithi055 ceph-mon[24757]: mgrmap e3: y(active, since 1.00787s) 2024-05-20T22:26:26.737 INFO:teuthology.orchestra.run.smithi055.stderr:mgr not available, waiting (3/15)... 2024-05-20T22:26:29.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:29 smithi055 ceph-mon[24757]: mgrmap e4: y(active, since 2s) 2024-05-20T22:26:29.416 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:29.416 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: { 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "fsid": "dd0bace8-16f7-11ef-bc9a-c7b262605968", 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "health": { 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "checks": {}, 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mutes": [] 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum": [ 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 0 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "a" 2024-05-20T22:26:29.417 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "monmap": { 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osdmap": { 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-20T22:26:29.418 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "pgmap": { 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "fsmap": { 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "available": true, 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "modules": [ 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "iostat", 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "nfs", 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "restful" 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ], 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "servicemap": { 2024-05-20T22:26:29.420 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:26:29.421 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "modified": "2024-05-20T22:26:12.669349+0000", 2024-05-20T22:26:29.421 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:26:29.421 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: }, 2024-05-20T22:26:29.421 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-20T22:26:29.421 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: } 2024-05-20T22:26:29.940 INFO:teuthology.orchestra.run.smithi055.stderr:mgr is available 2024-05-20T22:26:30.157 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:30 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3193592526' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T22:26:30.535 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: [global] 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: fsid = dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: [mgr] 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: [osd] 2024-05-20T22:26:30.536 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-20T22:26:30.537 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-20T22:26:31.013 INFO:teuthology.orchestra.run.smithi055.stderr:Enabling cephadm module... 2024-05-20T22:26:31.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:31 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3890316551' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-20T22:26:31.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:31 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3890316551' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-20T22:26:32.272 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:32 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1385618160' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-05-20T22:26:32.272 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:32 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ignoring --setuser ceph since I am not root 2024-05-20T22:26:32.272 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:32 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ignoring --setgroup ceph since I am not root 2024-05-20T22:26:32.272 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:32 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:32.274+0000 7f4a4a641000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:26:32.605 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:32 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:32.409+0000 7f4a4a641000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:26:32.605 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:32 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:32.512+0000 7f4a4a641000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:26:32.909 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:32 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:32.841+0000 7f4a4a641000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:26:33.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:33 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1385618160' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-05-20T22:26:33.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:33 smithi055 ceph-mon[24757]: mgrmap e5: y(active, since 6s) 2024-05-20T22:26:33.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:32 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:32.930+0000 7f4a4a641000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:26:33.426 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: { 2024-05-20T22:26:33.426 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 5, 2024-05-20T22:26:33.426 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "available": true, 2024-05-20T22:26:33.426 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "active_name": "y", 2024-05-20T22:26:33.426 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-20T22:26:33.426 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: } 2024-05-20T22:26:33.981 INFO:teuthology.orchestra.run.smithi055.stderr:Waiting for the mgr to restart... 2024-05-20T22:26:33.981 INFO:teuthology.orchestra.run.smithi055.stderr:Waiting for mgr epoch 5... 2024-05-20T22:26:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:34 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1317653475' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-20T22:26:34.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:34 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:34.147+0000 7f4a4a641000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:26:34.488 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:34 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:34.321+0000 7f4a4a641000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:26:34.488 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:34 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:34.410+0000 7f4a4a641000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:26:34.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:34 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:34.737+0000 7f4a4a641000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:26:34.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:34 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:34.831+0000 7f4a4a641000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:26:34.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:34 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:34.928+0000 7f4a4a641000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:26:35.488 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:35 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:35.178+0000 7f4a4a641000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:26:35.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:35 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:35.641+0000 7f4a4a641000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:26:36.488 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:36.155+0000 7f4a4a641000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:26:36.488 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:36.254+0000 7f4a4a641000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:26:36.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:36.563+0000 7f4a4a641000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:26:36.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:36.656+0000 7f4a4a641000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:26:37.433 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:37.081+0000 7f4a4a641000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:26:37.434 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:37.219+0000 7f4a4a641000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:26:37.434 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:37.316+0000 7f4a4a641000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:26:37.434 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:37.436+0000 7f4a4a641000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:26:37.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:37.529+0000 7f4a4a641000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:26:38.488 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:38 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:38.102+0000 7f4a4a641000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:26:38.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:38 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:38.506+0000 7f4a4a641000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:26:38.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:38 smithi055 ceph-mon[24757]: Active manager daemon y restarted 2024-05-20T22:26:38.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:38 smithi055 ceph-mon[24757]: Activating manager daemon y 2024-05-20T22:26:38.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:38 smithi055 ceph-mon[24757]: osdmap e2: 0 total, 0 up, 0 in 2024-05-20T22:26:39.575 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: { 2024-05-20T22:26:39.575 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-05-20T22:26:39.575 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "initialized": true 2024-05-20T22:26:39.575 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: } 2024-05-20T22:26:39.877 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: mgrmap e6: y(active, starting, since 0.0523951s) 2024-05-20T22:26:39.877 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: Manager daemon y is now available 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:26:39.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:39 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:26:40.131 INFO:teuthology.orchestra.run.smithi055.stderr:mgr epoch 5 is available 2024-05-20T22:26:40.131 INFO:teuthology.orchestra.run.smithi055.stderr:Setting orchestrator backend to cephadm... 2024-05-20T22:26:40.427 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:26:40] ENGINE Bus STARTING 2024-05-20T22:26:40.427 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:26:40] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:26:40.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:26:40] ENGINE Bus STARTED 2024-05-20T22:26:40.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:40 smithi055 ceph-mon[24757]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-05-20T22:26:40.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:40 smithi055 ceph-mon[24757]: mgrmap e7: y(active, since 1.06337s) 2024-05-20T22:26:40.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:40 smithi055 ceph-mon[24757]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-05-20T22:26:40.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:40 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:40.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:40 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:41.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:41 smithi055 ceph-mon[24757]: [20/May/2024:22:26:40] ENGINE Bus STARTING 2024-05-20T22:26:41.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:41 smithi055 ceph-mon[24757]: [20/May/2024:22:26:40] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:26:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:41 smithi055 ceph-mon[24757]: [20/May/2024:22:26:40] ENGINE Bus STARTED 2024-05-20T22:26:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:41 smithi055 ceph-mon[24757]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:41 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:41 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:41 smithi055 ceph-mon[24757]: mgrmap e8: y(active, since 2s) 2024-05-20T22:26:41.983 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: value unchanged 2024-05-20T22:26:42.480 INFO:teuthology.orchestra.run.smithi055.stderr:Generating ssh key... 2024-05-20T22:26:43.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:43 smithi055 ceph-mon[24757]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:43.488 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: Generating public/private rsa key pair. 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: Your identification has been saved in /tmp/tmpe4m6vud1/key. 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: Your public key has been saved in /tmp/tmpe4m6vud1/key.pub. 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: The key fingerprint is: 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: SHA256:6DXe0ZNAu0/9RiTa+4UHM3eEIqZvV3Xydq3T+VbG1V0 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: The key's randomart image is: 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: +---[RSA 3072]----+ 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: | . | 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: | . . .E| 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: | = . + O| 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: | . o = * B*| 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: | . S o * B.X| 2024-05-20T22:26:43.489 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: | . o + + o &B| 2024-05-20T22:26:43.490 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: | . . + o =oB| 2024-05-20T22:26:43.490 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: | . . =+| 2024-05-20T22:26:43.490 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: | .o| 2024-05-20T22:26:43.490 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: +----[SHA256]-----+ 2024-05-20T22:26:44.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:44 smithi055 ceph-mon[24757]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:44.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:44 smithi055 ceph-mon[24757]: Generating ssh key... 2024-05-20T22:26:44.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:44 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:44.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:44 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:44.402 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJHKirH81rS2psC+MuUis/noLsH6KxULJmAQWZByGVMp3by+KB0AoHbphHNfVI3skwnZMyYdDDqM8QE5WpI46RP3G6UZ2+AuX82SnzFTijxe2o2m/27lqNtkFlsnbGd1PUB3CTcZtcTCzuy1AcEfwEZDwzfj9fFdCQQVPfR3Y5mO3ta392Y5hQTe1g2woeP9CpmsGPh2U5gnQ0zAeEDl4ClM5pNZAXy/95dZc3k8BkvzrBAIxll2L9KBKUg5Uo/yGR+jCbZdVOM/Q6Snx2DbfHt8uta5JLh+yN5bOuKRen1uzsHi+31CnA21FnQ/R5XZSdEkPhSInesQin346aaH/0D7GyCq21z2pG9tEruvB/b1ayUFQydWeNzTGm7rHmfANTg7u2EmG8yypFiODe4cm8XoS/yo+ozi6kJh7E6EYIHj0TBjZytGmOW/znPMTDKc/RtxEQGHRCes84a0zZeAuuqCFWjBi+ZhGOPIx1PX2uKZUCr0N0L5JuOg5i0u1AST8= ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:26:44.913 INFO:teuthology.orchestra.run.smithi055.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-05-20T22:26:44.913 INFO:teuthology.orchestra.run.smithi055.stderr:Adding key to root@localhost authorized_keys... 2024-05-20T22:26:44.913 INFO:teuthology.orchestra.run.smithi055.stderr:Adding host smithi055... 2024-05-20T22:26:45.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:45 smithi055 ceph-mon[24757]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:46.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:46 smithi055 ceph-mon[24757]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi055", "addr": "172.21.15.55", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:46.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:46 smithi055 ceph-mon[24757]: Deploying cephadm binary to smithi055 2024-05-20T22:26:46.518 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: Added host 'smithi055' with addr '172.21.15.55' 2024-05-20T22:26:47.137 INFO:teuthology.orchestra.run.smithi055.stderr:Deploying unmanaged mon service... 2024-05-20T22:26:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:47 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:47 smithi055 ceph-mon[24757]: Added host smithi055 2024-05-20T22:26:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:47 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:47.786 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: Scheduled mon update... 2024-05-20T22:26:48.545 INFO:teuthology.orchestra.run.smithi055.stderr:Deploying unmanaged mgr service... 2024-05-20T22:26:48.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:48 smithi055 ceph-mon[24757]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:48.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:48 smithi055 ceph-mon[24757]: Saving service mon spec with placement count:5 2024-05-20T22:26:48.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:48 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:49.266 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-05-20T22:26:50.367 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:50 smithi055 ceph-mon[24757]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:50.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:50 smithi055 ceph-mon[24757]: Saving service mgr spec with placement count:2 2024-05-20T22:26:50.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:50 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:50.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:50 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:51.667 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:51 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:51.667 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:51 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1117239765' entity='client.admin' 2024-05-20T22:26:53.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:53 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:53.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:53 smithi055 ceph-mon[24757]: from='mgr.14120 172.21.15.55:0/830610926' entity='mgr.y' 2024-05-20T22:26:53.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:53 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1854277320' entity='client.admin' 2024-05-20T22:26:53.590 INFO:teuthology.orchestra.run.smithi055.stderr:Enabling the dashboard module... 2024-05-20T22:26:54.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:54 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2786913555' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-05-20T22:26:54.619 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:54 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ignoring --setuser ceph since I am not root 2024-05-20T22:26:54.619 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:54 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ignoring --setgroup ceph since I am not root 2024-05-20T22:26:54.619 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:54 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:54.488+0000 7fb5aef58000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:26:54.619 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:54 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:54.620+0000 7fb5aef58000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:26:54.980 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:54 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:54.722+0000 7fb5aef58000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:26:55.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:55 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:55.041+0000 7fb5aef58000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:26:55.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:55 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:55.127+0000 7fb5aef58000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:26:55.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:55 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2786913555' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-05-20T22:26:55.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:55 smithi055 ceph-mon[24757]: mgrmap e9: y(active, since 15s) 2024-05-20T22:26:56.445 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:56 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:56.328+0000 7fb5aef58000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:26:56.446 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: { 2024-05-20T22:26:56.446 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "epoch": 9, 2024-05-20T22:26:56.446 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "available": true, 2024-05-20T22:26:56.446 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "active_name": "y", 2024-05-20T22:26:56.447 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-20T22:26:56.447 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: } 2024-05-20T22:26:56.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:26:56 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/690618184' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-20T22:26:56.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:56 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:56.506+0000 7fb5aef58000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:26:56.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:56 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:56.592+0000 7fb5aef58000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:26:57.092 INFO:teuthology.orchestra.run.smithi055.stderr:Waiting for the mgr to restart... 2024-05-20T22:26:57.092 INFO:teuthology.orchestra.run.smithi055.stderr:Waiting for mgr epoch 9... 2024-05-20T22:26:57.217 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:56 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:56.919+0000 7fb5aef58000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:26:57.217 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:57.013+0000 7fb5aef58000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:26:57.217 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:57.111+0000 7fb5aef58000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:26:57.476 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:57.372+0000 7fb5aef58000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:26:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:57.836+0000 7fb5aef58000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:26:58.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:58 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:58.340+0000 7fb5aef58000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:26:58.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:58 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:58.437+0000 7fb5aef58000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:26:59.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:58 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:58.744+0000 7fb5aef58000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:26:59.254 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:58 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:58.836+0000 7fb5aef58000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:26:59.606 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:59 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:59.255+0000 7fb5aef58000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:26:59.606 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:59 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:59.392+0000 7fb5aef58000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:26:59.606 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:59 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:59.489+0000 7fb5aef58000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:26:59.606 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:59 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:59.607+0000 7fb5aef58000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:26:59.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:26:59 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:26:59.700+0000 7fb5aef58000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:27:00.641 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:27:00 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:27:00.270+0000 7fb5aef58000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:27:00.891 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:00 smithi055 ceph-mon[24757]: Active manager daemon y restarted 2024-05-20T22:27:00.891 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:00 smithi055 ceph-mon[24757]: Activating manager daemon y 2024-05-20T22:27:00.892 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:00 smithi055 ceph-mon[24757]: osdmap e3: 0 total, 0 up, 0 in 2024-05-20T22:27:00.892 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:27:00 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:27:00.643+0000 7fb5aef58000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:27:01.711 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: { 2024-05-20T22:27:01.712 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-05-20T22:27:01.712 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: "initialized": true 2024-05-20T22:27:01.712 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: } 2024-05-20T22:27:01.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:27:01 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:27:01] ENGINE Bus STARTING 2024-05-20T22:27:01.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: mgrmap e10: y(active, starting, since 0.0525861s) 2024-05-20T22:27:01.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:27:01.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:27:01.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:27:01.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:27:01.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:27:01.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: Manager daemon y is now available 2024-05-20T22:27:01.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:01.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:01.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:27:01.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:27:02.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:27:02 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:27:02] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:27:02.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:27:02 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:27:02] ENGINE Bus STARTED 2024-05-20T22:27:02.320 INFO:teuthology.orchestra.run.smithi055.stderr:mgr epoch 9 is available 2024-05-20T22:27:02.320 INFO:teuthology.orchestra.run.smithi055.stderr:Generating a dashboard self-signed certificate... 2024-05-20T22:27:02.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:02 smithi055 ceph-mon[24757]: mgrmap e11: y(active, since 1.06028s) 2024-05-20T22:27:02.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:02 smithi055 ceph-mon[24757]: [20/May/2024:22:27:01] ENGINE Bus STARTING 2024-05-20T22:27:02.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:02 smithi055 ceph-mon[24757]: [20/May/2024:22:27:02] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:27:02.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:02 smithi055 ceph-mon[24757]: [20/May/2024:22:27:02] ENGINE Bus STARTED 2024-05-20T22:27:02.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:02 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:02.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:02 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:03.094 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: Self-signed certificate created 2024-05-20T22:27:03.661 INFO:teuthology.orchestra.run.smithi055.stderr:Creating initial admin user... 2024-05-20T22:27:04.275 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:04 smithi055 ceph-mon[24757]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:04.275 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:04 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:04.275 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:04 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:04.275 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:04 smithi055 ceph-mon[24757]: mgrmap e12: y(active, since 2s) 2024-05-20T22:27:04.275 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:04 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:04.575 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$VaCICX5OlirfmerOi34HieXZJN7mYcd/hzgbW4YcYD799m7mA01RW", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1716244024, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-05-20T22:27:05.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:05 smithi055 ceph-mon[24757]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:05.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:05 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:05.414 INFO:teuthology.orchestra.run.smithi055.stderr:Fetching dashboard port number... 2024-05-20T22:27:05.994 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: 8443 2024-05-20T22:27:06.284 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:06 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3769106103' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-05-20T22:27:06.961 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: disabled 2024-05-20T22:27:06.969 INFO:teuthology.orchestra.run.smithi055.stderr:systemctl: inactive 2024-05-20T22:27:06.970 INFO:teuthology.orchestra.run.smithi055.stderr:firewalld.service is not enabled 2024-05-20T22:27:06.970 INFO:teuthology.orchestra.run.smithi055.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-05-20T22:27:06.973 INFO:teuthology.orchestra.run.smithi055.stderr:Ceph Dashboard is now available at: 2024-05-20T22:27:06.973 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T22:27:06.973 INFO:teuthology.orchestra.run.smithi055.stderr: URL: https://smithi055.front.sepia.ceph.com:8443/ 2024-05-20T22:27:06.973 INFO:teuthology.orchestra.run.smithi055.stderr: User: admin 2024-05-20T22:27:06.973 INFO:teuthology.orchestra.run.smithi055.stderr: Password: b8kk9u9vxh 2024-05-20T22:27:06.973 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T22:27:06.973 INFO:teuthology.orchestra.run.smithi055.stderr:Enabling autotune for osd_memory_target 2024-05-20T22:27:08.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:08 smithi055 ceph-mon[24757]: mgrmap e13: y(active, since 6s) 2024-05-20T22:27:09.048 INFO:teuthology.orchestra.run.smithi055.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-05-20T22:27:09.671 INFO:teuthology.orchestra.run.smithi055.stderr:You can access the Ceph CLI with: 2024-05-20T22:27:09.671 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T22:27:09.671 INFO:teuthology.orchestra.run.smithi055.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:27:09.671 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T22:27:09.672 INFO:teuthology.orchestra.run.smithi055.stderr:Please consider enabling telemetry to help improve Ceph: 2024-05-20T22:27:09.672 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T22:27:09.672 INFO:teuthology.orchestra.run.smithi055.stderr: ceph telemetry on 2024-05-20T22:27:09.672 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T22:27:09.672 INFO:teuthology.orchestra.run.smithi055.stderr:For more information see: 2024-05-20T22:27:09.672 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T22:27:09.672 INFO:teuthology.orchestra.run.smithi055.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-05-20T22:27:09.672 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T22:27:09.672 INFO:teuthology.orchestra.run.smithi055.stderr:Bootstrap complete. 2024-05-20T22:27:09.713 INFO:tasks.cephadm:Fetching config... 2024-05-20T22:27:09.713 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:27:09.713 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-05-20T22:27:09.731 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-05-20T22:27:09.732 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:27:09.732 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-05-20T22:27:09.788 INFO:tasks.cephadm:Fetching mon keyring... 2024-05-20T22:27:09.788 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:27:09.788 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/keyring of=/dev/stdout 2024-05-20T22:27:09.856 INFO:tasks.cephadm:Fetching pub ssh key... 2024-05-20T22:27:09.857 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:27:09.857 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-05-20T22:27:09.913 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-05-20T22:27:09.913 DEBUG:teuthology.orchestra.run.smithi055:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJHKirH81rS2psC+MuUis/noLsH6KxULJmAQWZByGVMp3by+KB0AoHbphHNfVI3skwnZMyYdDDqM8QE5WpI46RP3G6UZ2+AuX82SnzFTijxe2o2m/27lqNtkFlsnbGd1PUB3CTcZtcTCzuy1AcEfwEZDwzfj9fFdCQQVPfR3Y5mO3ta392Y5hQTe1g2woeP9CpmsGPh2U5gnQ0zAeEDl4ClM5pNZAXy/95dZc3k8BkvzrBAIxll2L9KBKUg5Uo/yGR+jCbZdVOM/Q6Snx2DbfHt8uta5JLh+yN5bOuKRen1uzsHi+31CnA21FnQ/R5XZSdEkPhSInesQin346aaH/0D7GyCq21z2pG9tEruvB/b1ayUFQydWeNzTGm7rHmfANTg7u2EmG8yypFiODe4cm8XoS/yo+ozi6kJh7E6EYIHj0TBjZytGmOW/znPMTDKc/RtxEQGHRCes84a0zZeAuuqCFWjBi+ZhGOPIx1PX2uKZUCr0N0L5JuOg5i0u1AST8= ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-20T22:27:09.989 INFO:teuthology.orchestra.run.smithi055.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJHKirH81rS2psC+MuUis/noLsH6KxULJmAQWZByGVMp3by+KB0AoHbphHNfVI3skwnZMyYdDDqM8QE5WpI46RP3G6UZ2+AuX82SnzFTijxe2o2m/27lqNtkFlsnbGd1PUB3CTcZtcTCzuy1AcEfwEZDwzfj9fFdCQQVPfR3Y5mO3ta392Y5hQTe1g2woeP9CpmsGPh2U5gnQ0zAeEDl4ClM5pNZAXy/95dZc3k8BkvzrBAIxll2L9KBKUg5Uo/yGR+jCbZdVOM/Q6Snx2DbfHt8uta5JLh+yN5bOuKRen1uzsHi+31CnA21FnQ/R5XZSdEkPhSInesQin346aaH/0D7GyCq21z2pG9tEruvB/b1ayUFQydWeNzTGm7rHmfANTg7u2EmG8yypFiODe4cm8XoS/yo+ozi6kJh7E6EYIHj0TBjZytGmOW/znPMTDKc/RtxEQGHRCes84a0zZeAuuqCFWjBi+ZhGOPIx1PX2uKZUCr0N0L5JuOg5i0u1AST8= ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:27:10.001 DEBUG:teuthology.orchestra.run.smithi138:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJHKirH81rS2psC+MuUis/noLsH6KxULJmAQWZByGVMp3by+KB0AoHbphHNfVI3skwnZMyYdDDqM8QE5WpI46RP3G6UZ2+AuX82SnzFTijxe2o2m/27lqNtkFlsnbGd1PUB3CTcZtcTCzuy1AcEfwEZDwzfj9fFdCQQVPfR3Y5mO3ta392Y5hQTe1g2woeP9CpmsGPh2U5gnQ0zAeEDl4ClM5pNZAXy/95dZc3k8BkvzrBAIxll2L9KBKUg5Uo/yGR+jCbZdVOM/Q6Snx2DbfHt8uta5JLh+yN5bOuKRen1uzsHi+31CnA21FnQ/R5XZSdEkPhSInesQin346aaH/0D7GyCq21z2pG9tEruvB/b1ayUFQydWeNzTGm7rHmfANTg7u2EmG8yypFiODe4cm8XoS/yo+ozi6kJh7E6EYIHj0TBjZytGmOW/znPMTDKc/RtxEQGHRCes84a0zZeAuuqCFWjBi+ZhGOPIx1PX2uKZUCr0N0L5JuOg5i0u1AST8= ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-20T22:27:10.044 INFO:teuthology.orchestra.run.smithi138.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJHKirH81rS2psC+MuUis/noLsH6KxULJmAQWZByGVMp3by+KB0AoHbphHNfVI3skwnZMyYdDDqM8QE5WpI46RP3G6UZ2+AuX82SnzFTijxe2o2m/27lqNtkFlsnbGd1PUB3CTcZtcTCzuy1AcEfwEZDwzfj9fFdCQQVPfR3Y5mO3ta392Y5hQTe1g2woeP9CpmsGPh2U5gnQ0zAeEDl4ClM5pNZAXy/95dZc3k8BkvzrBAIxll2L9KBKUg5Uo/yGR+jCbZdVOM/Q6Snx2DbfHt8uta5JLh+yN5bOuKRen1uzsHi+31CnA21FnQ/R5XZSdEkPhSInesQin346aaH/0D7GyCq21z2pG9tEruvB/b1ayUFQydWeNzTGm7rHmfANTg7u2EmG8yypFiODe4cm8XoS/yo+ozi6kJh7E6EYIHj0TBjZytGmOW/znPMTDKc/RtxEQGHRCes84a0zZeAuuqCFWjBi+ZhGOPIx1PX2uKZUCr0N0L5JuOg5i0u1AST8= ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:27:10.057 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-05-20T22:27:10.079 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:10 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/583871412' entity='client.admin' 2024-05-20T22:27:13.646 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-05-20T22:27:13.646 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-05-20T22:27:14.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:13 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/524110422' entity='client.admin' 2024-05-20T22:27:15.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:15 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:15.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:15 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi055", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:27:15.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:15 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:15.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:15 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:15.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:15 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:16.879 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi138 2024-05-20T22:27:16.879 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-05-20T22:27:16.880 DEBUG:teuthology.orchestra.run.smithi138:> dd of=/etc/ceph/ceph.conf 2024-05-20T22:27:16.899 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-05-20T22:27:16.899 DEBUG:teuthology.orchestra.run.smithi138:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:27:16.957 INFO:tasks.cephadm:Adding host smithi138 to orchestrator... 2024-05-20T22:27:16.957 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch host add smithi138 2024-05-20T22:27:17.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:17 smithi055 ceph-mon[24757]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:17.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:17.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:17.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:17.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:17.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:17 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:27:17.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:17.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:18.211 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:18 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:27:19.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:19 smithi055 ceph-mon[24757]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi138", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:19.945 INFO:teuthology.orchestra.run.smithi055.stdout:Added host 'smithi138' with addr '172.21.15.138' 2024-05-20T22:27:20.537 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch host ls --format=json 2024-05-20T22:27:21.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:20 smithi055 ceph-mon[24757]: Deploying cephadm binary to smithi138 2024-05-20T22:27:21.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:21.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:20 smithi055 ceph-mon[24757]: Added host smithi138 2024-05-20T22:27:21.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:21.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:21.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:21.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:21.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:22.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:21 smithi055 ceph-mon[24757]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:22.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:21 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:22.342 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:27:22.343 INFO:teuthology.orchestra.run.smithi055.stdout:[{"addr": "172.21.15.55", "hostname": "smithi055", "labels": [], "status": ""}, {"addr": "172.21.15.138", "hostname": "smithi138", "labels": [], "status": ""}] 2024-05-20T22:27:22.933 INFO:tasks.cephadm:Setting crush tunables to default 2024-05-20T22:27:22.933 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd crush tunables default 2024-05-20T22:27:23.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:23 smithi055 ceph-mon[24757]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:27:24.204 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:24 smithi055 ceph-mon[24757]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:25.308 INFO:teuthology.orchestra.run.smithi055.stderr:adjusted tunables profile to default 2024-05-20T22:27:25.603 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:25 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:25.603 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:25 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:25.603 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:25 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi138", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:27:25.603 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:25 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:25.603 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:25 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1151868055' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-20T22:27:25.974 INFO:tasks.cephadm:Adding mon.a on smithi055 2024-05-20T22:27:25.975 INFO:tasks.cephadm:Adding mon.c on smithi055 2024-05-20T22:27:25.975 INFO:tasks.cephadm:Adding mon.b on smithi138 2024-05-20T22:27:25.975 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch apply mon '3;smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi138:172.21.15.138=b' 2024-05-20T22:27:26.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:26 smithi055 ceph-mon[24757]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:26.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:26 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1151868055' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-20T22:27:26.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:26 smithi055 ceph-mon[24757]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:27:27.792 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled mon update... 2024-05-20T22:27:28.431 DEBUG:teuthology.orchestra.run.smithi055:mon.c> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.c.service 2024-05-20T22:27:28.433 DEBUG:teuthology.orchestra.run.smithi138:mon.b> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.b.service 2024-05-20T22:27:28.436 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-20T22:27:28.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:27:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:28 smithi055 ceph-mon[24757]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:28 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:28 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:28 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:28 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:28 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:28 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:28 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:28 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:29.568 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:29 smithi055 ceph-mon[24757]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi138:172.21.15.138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:29.568 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:29 smithi055 ceph-mon[24757]: Saving service mon spec with placement smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi138:172.21.15.138=b;count:3 2024-05-20T22:27:29.568 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:29 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:27:29.569 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:29 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:27:29.569 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:29 smithi055 ceph-mon[24757]: Deploying daemon mon.c on smithi055 2024-05-20T22:27:30.329 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:27:30.329 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":1,"fsid":"dd0bace8-16f7-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:26:07.295242Z","created":"2024-05-20T22:26:07.295242Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3300","nonce":0},{"type":"v1","addr":"172.21.15.55:6789","nonce":0}]},"addr":"172.21.15.55:6789/0","public_addr":"172.21.15.55:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:27:30.331 INFO:teuthology.orchestra.run.smithi138.stderr:dumped monmap epoch 1 2024-05-20T22:27:30.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:30 smithi055 ceph-mon[24757]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:31.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 podman[29480]: 2024-05-20 22:27:31.221540498 +0000 UTC m=+0.128999159 container create 79201364d4c8617d9071063bdbc3c8275061aaac18f1b7e706c2a3a3ba68ab2d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, version=8, io.buildah.version=1.19.8, io.openshift.expose-services=, release=754, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, name=centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-type=git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-20T22:27:31.602 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/396088075' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:27:31.603 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 podman[29480]: 2024-05-20 22:27:31.482667406 +0000 UTC m=+0.390126055 container init 79201364d4c8617d9071063bdbc3c8275061aaac18f1b7e706c2a3a3ba68ab2d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_BRANCH=HEAD, RELEASE=HEAD, ceph=True, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.tags=base centos centos-stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream) 2024-05-20T22:27:31.603 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 podman[29480]: 2024-05-20 22:27:31.48630611 +0000 UTC m=+0.393764753 container start 79201364d4c8617d9071063bdbc3c8275061aaac18f1b7e706c2a3a3ba68ab2d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vcs-type=git, RELEASE=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, version=8, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., release=754, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870) 2024-05-20T22:27:31.603 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T22:27:31.603 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-20T22:27:31.603 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: pidfile_write: ignore empty --pid-file 2024-05-20T22:27:31.603 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: load: jerasure load: lrc 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: RocksDB version: 6.15.5 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Compile date Apr 18 2022 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: DB SUMMARY 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: DB Session ID: ZCVZUSKPOW701Q48GKAW 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: CURRENT file: CURRENT 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.error_if_exists: 0 2024-05-20T22:27:31.604 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.create_if_missing: 0 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.paranoid_checks: 1 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.env: 0x55ff34ec6860 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.fs: Posix File System 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.info_log: 0x55ff373fddc0 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.statistics: (nil) 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.use_fsync: 0 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_log_file_size: 0 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T22:27:31.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.allow_fallocate: 1 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.use_direct_reads: 0 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.db_log_dir: 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T22:27:31.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.write_buffer_manager: 0x55ff374ee2d0 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.unordered_write: 0 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T22:27:31.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.row_cache: None 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.wal_filter: None 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.preserve_deletes: 0 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.two_write_queues: 0 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.atomic_flush: 0 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T22:27:31.608 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.log_readahead_size: 0 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_background_jobs: 2 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_background_compactions: -1 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_subcompactions: 1 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T22:27:31.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_open_files: -1 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_background_flushes: -1 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Compression algorithms supported: 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: kZSTD supported: 0 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: kXpressCompression supported: 0 2024-05-20T22:27:31.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: kLZ4Compression supported: 1 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: kBZip2Compression supported: 0 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: kZlibCompression supported: 1 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: kSnappyCompression supported: 1 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.merge_operator: 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_filter: None 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T22:27:31.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ff373c9d00) 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: cache_index_and_filter_blocks: 1 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: pin_top_level_index_and_filter: 1 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_type: 0 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: data_block_index_type: 0 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_shortening: 1 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: hash_index_allow_collision: 1 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: checksum: 1 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: no_block_cache: 0 2024-05-20T22:27:31.612 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache: 0x55ff37434170 2024-05-20T22:27:31.613 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_name: BinnedLRUCache 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_options: 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: capacity : 536870912 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: num_shard_bits : 4 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: strict_capacity_limit : 0 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: high_pri_pool_ratio: 0.000 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_compressed: (nil) 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: persistent_cache: (nil) 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_size: 4096 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_size_deviation: 10 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_restart_interval: 16 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_block_restart_interval: 1 2024-05-20T22:27:31.614 INFO:journalctl@ceph.mon.c.smithi055.stdout: metadata_block_size: 4096 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout: partition_filters: 0 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_delta_encoding: 1 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout: whole_key_filtering: 1 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout: verify_compression: 0 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout: read_amp_bytes_per_bit: 0 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout: format_version: 4 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout: enable_index_compression: 1 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_align: 0 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compression: NoCompression 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T22:27:31.615 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.num_levels: 7 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T22:27:31.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T22:27:31.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.arena_block_size: 4194304 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T22:27:31.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.table_properties_collectors: 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.inplace_update_support: 0 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T22:27:31.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T22:27:31.620 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.bloom_locality: 0 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.max_successive_merges: 0 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.ttl: 2592000 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.enable_blob_files: false 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.min_blob_size: 0 2024-05-20T22:27:31.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716244051513605, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716244051514051, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1716244051, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "45cee8c3-08c7-45fc-bc80-49d57f8f1da6", "db_session_id": "ZCVZUSKPOW701Q48GKAW"}} 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716244051514609, "job": 1, "event": "recovery_finished"} 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T22:27:31.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55ff3741aa80 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: DB pointer 0x55ff3742a000 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** DB Stats ** 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:27:31.624 INFO:journalctl@ceph.mon.c.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:27:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-20T22:27:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:27:31.627 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: using public_addrv [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: starting mon.c rank -1 at public addrs [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] at bind addrs [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(???) e0 preinit fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).mds e1 new map 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).mds e1 print_map 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout: e1 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout: legacy client fscid: -1 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout: No filesystems configured 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-05-20T22:27:31.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1151868055' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:31.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi138:172.21.15.138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: Saving service mon spec with placement smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi138:172.21.15.138=b;count:3 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: Deploying daemon mon.c on smithi055 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/396088075' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:27:31.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta expand map: {default=false} 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta from 'false' to 'false' 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta expanded map: {default=false} 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta expand map: {default=info} 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta from 'info' to 'info' 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta expanded map: {default=info} 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta expand map: {default=daemon} 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta from 'daemon' to 'daemon' 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta expanded map: {default=daemon} 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta expand map: {default=debug} 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta from 'debug' to 'debug' 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: expand_channel_meta expanded map: {default=debug} 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 ceph-mon[29498]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 bash[29480]: 79201364d4c8617d9071063bdbc3c8275061aaac18f1b7e706c2a3a3ba68ab2d 2024-05-20T22:27:31.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:31 smithi055 systemd[1]: Started Ceph mon.c for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:27:31.857 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-20T22:27:31.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:27:36.890 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:27:36.890 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":2,"fsid":"dd0bace8-16f7-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:27:31.533148Z","created":"2024-05-20T22:26:07.295242Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3300","nonce":0},{"type":"v1","addr":"172.21.15.55:6789","nonce":0}]},"addr":"172.21.15.55:6789/0","public_addr":"172.21.15.55:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3301","nonce":0},{"type":"v1","addr":"172.21.15.55:6790","nonce":0}]},"addr":"172.21.15.55:6790/0","public_addr":"172.21.15.55:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-20T22:27:36.892 INFO:teuthology.orchestra.run.smithi138.stderr:dumped monmap epoch 2 2024-05-20T22:27:36.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:27:36.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: mon.a calling monitor election 2024-05-20T22:27:36.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:36.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: mon.c calling monitor election 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: monmap e2: 2 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: fsmap 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: mgrmap e13: y(active, since 35s) 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: overall HEALTH_OK 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:36.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: mon.a calling monitor election 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: mon.c calling monitor election 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:36.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-20T22:27:36.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: monmap e2: 2 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:27:36.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: fsmap 2024-05-20T22:27:36.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:27:36.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: mgrmap e13: y(active, since 35s) 2024-05-20T22:27:36.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: overall HEALTH_OK 2024-05-20T22:27:36.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:36.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:36.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:37.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:37 smithi055 ceph-mon[24757]: Deploying daemon mon.b on smithi138 2024-05-20T22:27:37.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:37 smithi055 ceph-mon[24757]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:37.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:37 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/4036919215' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:27:37.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:37 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:37.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:37 smithi055 ceph-mon[29498]: Deploying daemon mon.b on smithi138 2024-05-20T22:27:37.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:37 smithi055 ceph-mon[29498]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:37.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:37 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/4036919215' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:27:37.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:37 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:38.724 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-20T22:27:38.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:27:40.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:39 smithi055 ceph-mon[24757]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:40.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:39 smithi055 ceph-mon[29498]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:41.354 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:27:41.354 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":2,"fsid":"dd0bace8-16f7-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:27:31.533148Z","created":"2024-05-20T22:26:07.295242Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3300","nonce":0},{"type":"v1","addr":"172.21.15.55:6789","nonce":0}]},"addr":"172.21.15.55:6789/0","public_addr":"172.21.15.55:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3301","nonce":0},{"type":"v1","addr":"172.21.15.55:6790","nonce":0}]},"addr":"172.21.15.55:6790/0","public_addr":"172.21.15.55:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-20T22:27:41.355 INFO:teuthology.orchestra.run.smithi138.stderr:dumped monmap epoch 2 2024-05-20T22:27:42.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:41 smithi055 ceph-mon[24757]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:42.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:41 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/3425589427' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:27:42.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:41 smithi055 ceph-mon[29498]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:42.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:41 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/3425589427' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:27:43.342 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-20T22:27:43.342 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:27:47.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:27:47.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: mon.a calling monitor election 2024-05-20T22:27:47.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: mon.c calling monitor election 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: fsmap 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: mgrmap e13: y(active, since 46s) 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: mon.b (rank 2) addr [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] is down (out of quorum) 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:47.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: mon.a calling monitor election 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: mon.c calling monitor election 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-20T22:27:47.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: fsmap 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: mgrmap e13: y(active, since 46s) 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: mon.b (rank 2) addr [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] is down (out of quorum) 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:47.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:48.182 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-05-20T22:27:48.182 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":3,"fsid":"dd0bace8-16f7-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:27:41.824528Z","created":"2024-05-20T22:26:07.295242Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3300","nonce":0},{"type":"v1","addr":"172.21.15.55:6789","nonce":0}]},"addr":"172.21.15.55:6789/0","public_addr":"172.21.15.55:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3301","nonce":0},{"type":"v1","addr":"172.21.15.55:6790","nonce":0}]},"addr":"172.21.15.55:6790/0","public_addr":"172.21.15.55:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:3300","nonce":0},{"type":"v1","addr":"172.21.15.138:6789","nonce":0}]},"addr":"172.21.15.138:6789/0","public_addr":"172.21.15.138:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-20T22:27:48.183 INFO:teuthology.orchestra.run.smithi138.stderr:dumped monmap epoch 3 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:27:48.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:27:48.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:48.736 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-05-20T22:27:48.736 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph config generate-minimal-conf 2024-05-20T22:27:49.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:49.428 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: mon.b calling monitor election 2024-05-20T22:27:49.428 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: mon.b calling monitor election 2024-05-20T22:27:49.428 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: mon.a calling monitor election 2024-05-20T22:27:49.428 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: mon.c calling monitor election 2024-05-20T22:27:49.428 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:27:49.428 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:27:49.428 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: fsmap 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: mgrmap e13: y(active, since 48s) 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: Cluster is now healthy 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[24757]: overall HEALTH_OK 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: mon.b calling monitor election 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: mon.b calling monitor election 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: mon.a calling monitor election 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: mon.c calling monitor election 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:27:49.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: fsmap 2024-05-20T22:27:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:27:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: mgrmap e13: y(active, since 48s) 2024-05-20T22:27:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-05-20T22:27:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: Cluster is now healthy 2024-05-20T22:27:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:49 smithi055 ceph-mon[29498]: overall HEALTH_OK 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: mon.b calling monitor election 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: mon.b calling monitor election 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: mon.a calling monitor election 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: mon.c calling monitor election 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: fsmap 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: mgrmap e13: y(active, since 48s) 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: Cluster is now healthy 2024-05-20T22:27:49.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:49 smithi138 ceph-mon[25673]: overall HEALTH_OK 2024-05-20T22:27:50.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[24757]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:50.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:50.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:50.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:27:50.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:50.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:50.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[29498]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:50.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:50.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:50.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:27:50.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:50.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:50 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:50.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:50 smithi138 ceph-mon[25673]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:50.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:50 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:50.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:50 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:50.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:50 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:27:50.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:50 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:50.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:50 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:27:51.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:51 smithi055 ceph-mon[24757]: Reconfiguring mon.c (monmap changed)... 2024-05-20T22:27:51.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:51 smithi055 ceph-mon[24757]: Reconfiguring daemon mon.c on smithi055 2024-05-20T22:27:51.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:51 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:51.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:51 smithi055 ceph-mon[29498]: Reconfiguring mon.c (monmap changed)... 2024-05-20T22:27:51.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:51 smithi055 ceph-mon[29498]: Reconfiguring daemon mon.c on smithi055 2024-05-20T22:27:51.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:51 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:51.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:51 smithi138 ceph-mon[25673]: Reconfiguring mon.c (monmap changed)... 2024-05-20T22:27:51.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:51 smithi138 ceph-mon[25673]: Reconfiguring daemon mon.c on smithi055 2024-05-20T22:27:51.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:51 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:52.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:52 smithi138 ceph-mon[25673]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:52.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:52 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:52.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:52 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:52.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:52 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:27:52.272 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:52 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:52.342 INFO:teuthology.orchestra.run.smithi055.stdout:# minimal ceph.conf for dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:27:52.343 INFO:teuthology.orchestra.run.smithi055.stdout:[global] 2024-05-20T22:27:52.343 INFO:teuthology.orchestra.run.smithi055.stdout: fsid = dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:27:52.343 INFO:teuthology.orchestra.run.smithi055.stdout: mon_host = [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] 2024-05-20T22:27:52.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[24757]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:52.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:52.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:52.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:27:52.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:52.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[29498]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:52.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:52.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:27:52.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:27:52.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:52 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:52.956 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-05-20T22:27:52.956 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:27:52.956 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/ceph/ceph.conf 2024-05-20T22:27:52.985 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:27:52.986 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:27:53.051 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-05-20T22:27:53.051 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.conf 2024-05-20T22:27:53.078 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-05-20T22:27:53.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:27:53.144 INFO:tasks.cephadm:Adding mgr.y on smithi055 2024-05-20T22:27:53.144 INFO:tasks.cephadm:Adding mgr.x on smithi138 2024-05-20T22:27:53.144 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch apply mgr '2;smithi055=y;smithi138=x' 2024-05-20T22:27:53.205 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:53 smithi138 ceph-mon[25673]: Reconfiguring mon.b (monmap changed)... 2024-05-20T22:27:53.206 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:53 smithi138 ceph-mon[25673]: Reconfiguring daemon mon.b on smithi138 2024-05-20T22:27:53.206 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:53 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1000381149' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:53.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:53 smithi055 ceph-mon[24757]: Reconfiguring mon.b (monmap changed)... 2024-05-20T22:27:53.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:53 smithi055 ceph-mon[24757]: Reconfiguring daemon mon.b on smithi138 2024-05-20T22:27:53.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:53 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1000381149' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:53.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:53 smithi055 ceph-mon[29498]: Reconfiguring mon.b (monmap changed)... 2024-05-20T22:27:53.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:53 smithi055 ceph-mon[29498]: Reconfiguring daemon mon.b on smithi138 2024-05-20T22:27:53.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:53 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1000381149' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:54.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:54 smithi138 ceph-mon[25673]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:54.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:54 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:54.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:54 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:54.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:54 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:54.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:54 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:54.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:54 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:54.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[24757]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:54.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:54.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:54.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:54.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:54.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:54.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[29498]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:54.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:54.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:54.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:54.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:54.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:54 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:54.875 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled mgr update... 2024-05-20T22:27:55.488 DEBUG:teuthology.orchestra.run.smithi138:mgr.x> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.x.service 2024-05-20T22:27:55.531 INFO:tasks.cephadm:Deploying OSDs... 2024-05-20T22:27:55.531 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:27:55.531 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/scratch_devs of=/dev/stdout 2024-05-20T22:27:55.552 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-20T22:27:55.552 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_1 2024-05-20T22:27:55.612 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-20T22:27:55.612 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:27:55.612 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-05-20T22:27:55.613 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:27:55.613 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:27:55.613 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2024-05-20 22:27:30.923089084 +0000 2024-05-20T22:27:55.613 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2024-05-20 22:24:48.387818123 +0000 2024-05-20T22:27:55.613 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2024-05-20 22:24:48.387818123 +0000 2024-05-20T22:27:55.613 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: 2024-05-20 22:24:48.387818123 +0000 2024-05-20T22:27:55.613 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-20T22:27:55.681 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2024-05-20T22:27:55.681 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2024-05-20T22:27:55.682 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000306843 s, 1.7 MB/s 2024-05-20T22:27:55.683 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-20T22:27:55.744 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_2 2024-05-20T22:27:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-20T22:27:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:27:55.803 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-05-20T22:27:55.803 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:27:55.803 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:27:55.803 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2024-05-20 22:27:30.924089064 +0000 2024-05-20T22:27:55.803 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2024-05-20 22:24:48.735811184 +0000 2024-05-20T22:27:55.804 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2024-05-20 22:24:48.735811184 +0000 2024-05-20T22:27:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: 2024-05-20 22:24:48.735811184 +0000 2024-05-20T22:27:55.804 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-20T22:27:55.872 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2024-05-20T22:27:55.872 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2024-05-20T22:27:55.872 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000305202 s, 1.7 MB/s 2024-05-20T22:27:55.873 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-20T22:27:55.933 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_3 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='client.34109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi055=y;smithi138=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: Saving service mgr spec with placement smithi055=y;smithi138=x;count:2 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:27:55.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T22:27:55.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:27:55.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:55.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[24757]: Deploying daemon mgr.x on smithi138 2024-05-20T22:27:55.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:55.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='client.34109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi055=y;smithi138=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:55.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: Saving service mgr spec with placement smithi055=y;smithi138=x;count:2 2024-05-20T22:27:55.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:55.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:55.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:55.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:55.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:55.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:27:55.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T22:27:55.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:27:55.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:55.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:55 smithi055 ceph-mon[29498]: Deploying daemon mgr.x on smithi138 2024-05-20T22:27:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-20T22:27:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:27:55.997 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-05-20T22:27:55.997 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:27:55.997 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:27:55.997 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2024-05-20 22:27:30.924089064 +0000 2024-05-20T22:27:55.997 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2024-05-20 22:24:49.094804026 +0000 2024-05-20T22:27:55.997 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2024-05-20 22:24:49.094804026 +0000 2024-05-20T22:27:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: 2024-05-20 22:24:49.094804026 +0000 2024-05-20T22:27:55.997 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-20T22:27:56.065 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2024-05-20T22:27:56.065 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2024-05-20T22:27:56.065 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000361621 s, 1.4 MB/s 2024-05-20T22:27:56.066 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-20T22:27:56.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:56.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='client.34109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi055=y;smithi138=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: Saving service mgr spec with placement smithi055=y;smithi138=x;count:2 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:27:56.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:56.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:55 smithi138 ceph-mon[25673]: Deploying daemon mgr.x on smithi138 2024-05-20T22:27:56.129 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_4 2024-05-20T22:27:56.188 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-20T22:27:56.188 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:27:56.188 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-05-20T22:27:56.188 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:27:56.188 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:27:56.188 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2024-05-20 22:27:30.925089044 +0000 2024-05-20T22:27:56.188 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2024-05-20 22:24:49.437797187 +0000 2024-05-20T22:27:56.188 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2024-05-20 22:24:49.437797187 +0000 2024-05-20T22:27:56.188 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: 2024-05-20 22:24:49.437797187 +0000 2024-05-20T22:27:56.189 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-20T22:27:56.255 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2024-05-20T22:27:56.256 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2024-05-20T22:27:56.256 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000380343 s, 1.3 MB/s 2024-05-20T22:27:56.257 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-20T22:27:56.315 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-05-20T22:27:56.316 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/scratch_devs of=/dev/stdout 2024-05-20T22:27:56.334 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-20T22:27:56.334 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_1 2024-05-20T22:27:56.391 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-20T22:27:56.391 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:27:56.391 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-05-20T22:27:56.391 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:27:56.392 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:27:56.392 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2024-05-20 22:27:40.324927618 +0000 2024-05-20T22:27:56.392 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2024-05-20 22:24:48.383304905 +0000 2024-05-20T22:27:56.392 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2024-05-20 22:24:48.383304905 +0000 2024-05-20T22:27:56.392 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: 2024-05-20 22:24:48.383304905 +0000 2024-05-20T22:27:56.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-20T22:27:56.459 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2024-05-20T22:27:56.459 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2024-05-20T22:27:56.459 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000221328 s, 2.3 MB/s 2024-05-20T22:27:56.460 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-20T22:27:56.519 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_2 2024-05-20T22:27:56.577 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-20T22:27:56.577 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:27:56.577 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-05-20T22:27:56.577 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:27:56.577 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:27:56.577 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2024-05-20 22:27:40.325927597 +0000 2024-05-20T22:27:56.577 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2024-05-20 22:24:48.730297359 +0000 2024-05-20T22:27:56.577 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2024-05-20 22:24:48.730297359 +0000 2024-05-20T22:27:56.578 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: 2024-05-20 22:24:48.730297359 +0000 2024-05-20T22:27:56.578 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-20T22:27:56.762 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2024-05-20T22:27:56.762 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2024-05-20T22:27:56.762 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000200959 s, 2.5 MB/s 2024-05-20T22:27:56.763 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-20T22:27:56.779 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_3 2024-05-20T22:27:56.840 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-20T22:27:56.840 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:27:56.840 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-05-20T22:27:56.840 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:27:56.840 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:27:56.841 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2024-05-20 22:27:56.724572401 +0000 2024-05-20T22:27:56.841 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2024-05-20 22:24:49.068290009 +0000 2024-05-20T22:27:56.841 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2024-05-20 22:24:49.068290009 +0000 2024-05-20T22:27:56.841 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: 2024-05-20 22:24:49.068290009 +0000 2024-05-20T22:27:56.841 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-20T22:27:56.939 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2024-05-20T22:27:56.939 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2024-05-20T22:27:56.939 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000185159 s, 2.8 MB/s 2024-05-20T22:27:56.940 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-20T22:27:56.956 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_4 2024-05-20T22:27:57.013 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-20T22:27:57.013 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:27:57.013 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-05-20T22:27:57.013 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:27:57.013 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:27:57.013 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2024-05-20 22:27:56.902568544 +0000 2024-05-20T22:27:57.013 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2024-05-20 22:24:49.417282419 +0000 2024-05-20T22:27:57.014 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2024-05-20 22:24:49.417282419 +0000 2024-05-20T22:27:57.014 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: 2024-05-20 22:24:49.417282419 +0000 2024-05-20T22:27:57.014 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-20T22:27:57.075 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:56 smithi138 systemd[1]: Starting Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:27:57.078 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2024-05-20T22:27:57.078 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2024-05-20T22:27:57.078 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000186012 s, 2.8 MB/s 2024-05-20T22:27:57.079 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-20T22:27:57.136 INFO:tasks.cephadm:Deploying osd.0 on smithi055 with /dev/vg_nvme/lv_4... 2024-05-20T22:27:57.137 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-20T22:27:57.353 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 podman[26795]: 2024-05-20 22:27:57.166807581 +0000 UTC m=+0.108187792 container create 6b6d8b70e257f3ed566c78020e708b2fbd4386a0a5efe97f362d008ef6e82dce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, version=8, build-date=2022-05-03T08:36:31.336870, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, maintainer=Guillaume Abrioux , RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, name=centos-stream, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, architecture=x86_64, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T22:27:57.353 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 podman[26795]: 2024-05-20 22:27:57.076491288 +0000 UTC m=+0.017871465 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-20T22:27:57.611 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 podman[26795]: 2024-05-20 22:27:57.357067818 +0000 UTC m=+0.298447984 container init 6b6d8b70e257f3ed566c78020e708b2fbd4386a0a5efe97f362d008ef6e82dce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, vcs-type=git, name=centos-stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, architecture=x86_64, GIT_BRANCH=HEAD) 2024-05-20T22:27:57.611 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 podman[26795]: 2024-05-20 22:27:57.36053692 +0000 UTC m=+0.301917090 container start 6b6d8b70e257f3ed566c78020e708b2fbd4386a0a5efe97f362d008ef6e82dce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, ceph=True, GIT_BRANCH=HEAD, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_CLEAN=True, architecture=x86_64, release=754, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux ) 2024-05-20T22:27:57.611 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 ceph-mgr[26851]: -- 172.21.15.138:0/2584351852 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5621d27e2340 con 0x5621d27ba800 2024-05-20T22:27:57.611 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 bash[26795]: 6b6d8b70e257f3ed566c78020e708b2fbd4386a0a5efe97f362d008ef6e82dce 2024-05-20T22:27:57.611 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 systemd[1]: Started Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:27:57.611 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:57.499+0000 7f41cc1c3000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:27:58.073 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:57.640+0000 7f41cc1c3000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:27:58.074 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:57 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:57.743+0000 7f41cc1c3000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:27:58.074 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:58 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:58.076+0000 7f41cc1c3000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:27:58.360 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:58 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:58.165+0000 7f41cc1c3000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:27:58.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:58 smithi138 ceph-mon[25673]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:58.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:58 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:58.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:58 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:58.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:58 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:58.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:58 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:58.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[24757]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:58.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:58.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:58.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:58.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[29498]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:58.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:58 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:59.313 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-20T22:27:59.313 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-20T22:27:59.313 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2024-05-20T22:27:59.313 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2024-05-20T22:27:59.313 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0158195 s, 663 MB/s 2024-05-20T22:27:59.313 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2024-05-20T22:27:59.610 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:59 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:59.332+0000 7f41cc1c3000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:27:59.610 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:59 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:59.497+0000 7f41cc1c3000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:27:59.611 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:59 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:59.577+0000 7f41cc1c3000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:27:59.919 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:59.919 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:59.919 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[24757]: Reconfiguring mgr.y (unknown last config time)... 2024-05-20T22:27:59.919 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[24757]: Reconfiguring daemon mgr.y on smithi055 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[24757]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[29498]: Reconfiguring mgr.y (unknown last config time)... 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[29498]: Reconfiguring daemon mgr.y on smithi055 2024-05-20T22:27:59.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:27:59 smithi055 ceph-mon[29498]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:27:59.974 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:59 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:59.888+0000 7f41cc1c3000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:27:59.974 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:59 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:59.975 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:59 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:27:59.975 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:59 smithi138 ceph-mon[25673]: Reconfiguring mgr.y (unknown last config time)... 2024-05-20T22:27:59.975 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:59 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:27:59.975 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:59 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:27:59.975 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:59 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:59.975 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:59 smithi138 ceph-mon[25673]: Reconfiguring daemon mgr.y on smithi055 2024-05-20T22:27:59.975 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:27:59 smithi138 ceph-mon[25673]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:00.232 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi055:vg_nvme/lv_4 2024-05-20T22:28:00.306 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:27:59 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:27:59.977+0000 7f41cc1c3000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:28:00.306 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:00 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:00.069+0000 7f41cc1c3000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:28:00.306 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:00 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:00.308+0000 7f41cc1c3000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:28:01.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:00 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:00.739+0000 7f41cc1c3000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:28:01.601 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:01.223+0000 7f41cc1c3000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:28:01.601 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:01.316+0000 7f41cc1c3000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:28:01.601 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:01.604+0000 7f41cc1c3000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:28:01.860 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:01.694+0000 7f41cc1c3000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:28:01.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:01.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:01.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:01.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:01.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:01.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:01 smithi138 ceph-mon[25673]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:01.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:28:01.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:28:01.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:01.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:01.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:01.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:01.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:01.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:01.920 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[24757]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[29498]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:01.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:28:01.922 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:28:01.922 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:02.360 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:02 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:02.094+0000 7f41cc1c3000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:28:02.360 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:02 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:02.225+0000 7f41cc1c3000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:28:02.360 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:02 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:02.318+0000 7f41cc1c3000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:28:02.860 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:02 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:02.431+0000 7f41cc1c3000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:28:02.861 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:02 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:02.517+0000 7f41cc1c3000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:28:03.360 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:03 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:03.053+0000 7f41cc1c3000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:28:03.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:03 smithi138 ceph-mon[25673]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:03.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:03 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:28:03.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:03 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:28:03.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:03 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:03.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:03 smithi055 ceph-mon[24757]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:03.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:03 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:28:03.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:03 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:28:03.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:03 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:03.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:03 smithi055 ceph-mon[29498]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:03.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:03 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:28:03.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:03 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:28:03.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:03 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:03.860 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:28:03 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:28:03.414+0000 7f41cc1c3000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:28:04.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[24757]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:04.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[24757]: Standby manager daemon x started 2024-05-20T22:28:04.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:28:04.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:28:04.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:28:04.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:28:04.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[29498]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:04.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[29498]: Standby manager daemon x started 2024-05-20T22:28:04.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:28:04.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:28:04.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:28:04.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:04 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:28:04.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:04 smithi138 ceph-mon[25673]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:04.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:04 smithi138 ceph-mon[25673]: Standby manager daemon x started 2024-05-20T22:28:04.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:04 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:28:04.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:04 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:28:04.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:04 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:28:04.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:04 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/3808282093' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:28:05.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[24757]: mgrmap e14: y(active, since 63s), standbys: x 2024-05-20T22:28:05.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:28:05.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2778152868' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32a19f07-edab-4360-a0df-a91d9239763e"}]: dispatch 2024-05-20T22:28:05.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2778152868' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "32a19f07-edab-4360-a0df-a91d9239763e"}]': finished 2024-05-20T22:28:05.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[24757]: osdmap e5: 1 total, 0 up, 1 in 2024-05-20T22:28:05.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:05.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[29498]: mgrmap e14: y(active, since 63s), standbys: x 2024-05-20T22:28:05.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:28:05.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2778152868' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32a19f07-edab-4360-a0df-a91d9239763e"}]: dispatch 2024-05-20T22:28:05.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2778152868' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "32a19f07-edab-4360-a0df-a91d9239763e"}]': finished 2024-05-20T22:28:05.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[29498]: osdmap e5: 1 total, 0 up, 1 in 2024-05-20T22:28:05.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:05 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:05.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:05 smithi138 ceph-mon[25673]: mgrmap e14: y(active, since 63s), standbys: x 2024-05-20T22:28:05.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:05 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:28:05.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:05 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2778152868' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "32a19f07-edab-4360-a0df-a91d9239763e"}]: dispatch 2024-05-20T22:28:05.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:05 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2778152868' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "32a19f07-edab-4360-a0df-a91d9239763e"}]': finished 2024-05-20T22:28:05.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:05 smithi138 ceph-mon[25673]: osdmap e5: 1 total, 0 up, 1 in 2024-05-20T22:28:05.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:05 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:06.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:06 smithi055 ceph-mon[24757]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:06.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:06 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/637879609' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:28:06.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:06 smithi055 ceph-mon[29498]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:06.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:06 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/637879609' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:28:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:06 smithi138 ceph-mon[25673]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:06.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:06 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/637879609' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:28:08.293 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:08 smithi055 ceph-mon[24757]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:08.294 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:08 smithi055 ceph-mon[29498]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:08.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:08 smithi138 ceph-mon[25673]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:10.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:10 smithi055 ceph-mon[24757]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:10.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:10 smithi055 ceph-mon[29498]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:10.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:10 smithi138 ceph-mon[25673]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:11.476 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:11 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:28:11.476 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:11 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:11.476 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:11 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:28:11.476 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:11 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:11.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:11 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:28:11.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:11 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:12.465 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:12 smithi055 ceph-mon[24757]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:12.465 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:12 smithi055 ceph-mon[24757]: Deploying daemon osd.0 on smithi055 2024-05-20T22:28:12.465 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:12 smithi055 ceph-mon[29498]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:12.466 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:12 smithi055 ceph-mon[29498]: Deploying daemon osd.0 on smithi055 2024-05-20T22:28:12.570 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:12 smithi138 ceph-mon[25673]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:12.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:12 smithi138 ceph-mon[25673]: Deploying daemon osd.0 on smithi055 2024-05-20T22:28:14.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:14 smithi055 ceph-mon[24757]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:14.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:14 smithi055 ceph-mon[29498]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:14.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:14 smithi138 ceph-mon[25673]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:16.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:16 smithi055 ceph-mon[24757]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:16.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:16 smithi055 ceph-mon[29498]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:16.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:16 smithi138 ceph-mon[25673]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:17.469 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:17.469 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:17.469 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:17.469 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:17 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:17.470 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:17 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:17.470 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:17 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:17.470 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:17 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:17.470 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:17 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:17.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:17 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:17.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:17 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:17.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:17 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:17.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:17 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:18.287 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:18 smithi055 ceph-mon[24757]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:18.288 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:18 smithi055 ceph-mon[29498]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:18.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:18 smithi138 ceph-mon[25673]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:19.819 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 0 on host 'smithi055' 2024-05-20T22:28:20.126 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[29498]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:20.126 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[29498]: from='osd.0 [v2:172.21.15.55:6802/42457402,v1:172.21.15.55:6803/42457402]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[29498]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[24757]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[24757]: from='osd.0 [v2:172.21.15.55:6802/42457402,v1:172.21.15.55:6803/42457402]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:28:20.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[24757]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:28:20.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:20.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:20.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:20.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:20.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:20.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:20 smithi138 ceph-mon[25673]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:20.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:20 smithi138 ceph-mon[25673]: from='osd.0 [v2:172.21.15.55:6802/42457402,v1:172.21.15.55:6803/42457402]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:28:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:20 smithi138 ceph-mon[25673]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:28:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:20.488 DEBUG:teuthology.orchestra.run.smithi055:osd.0> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.0.service 2024-05-20T22:28:20.491 INFO:tasks.cephadm:Deploying osd.1 on smithi055 with /dev/vg_nvme/lv_3... 2024-05-20T22:28:20.491 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-20T22:28:21.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:21 smithi138 ceph-mon[25673]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:28:21.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:21 smithi138 ceph-mon[25673]: osdmap e6: 1 total, 0 up, 1 in 2024-05-20T22:28:21.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:21 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:21.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:21 smithi138 ceph-mon[25673]: from='osd.0 [v2:172.21.15.55:6802/42457402,v1:172.21.15.55:6803/42457402]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:28:21.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:21 smithi138 ceph-mon[25673]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:28:21.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[29498]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:28:21.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[29498]: osdmap e6: 1 total, 0 up, 1 in 2024-05-20T22:28:21.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:21.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[29498]: from='osd.0 [v2:172.21.15.55:6802/42457402,v1:172.21.15.55:6803/42457402]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:28:21.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[29498]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:28:21.670 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[24757]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:28:21.670 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[24757]: osdmap e6: 1 total, 0 up, 1 in 2024-05-20T22:28:21.670 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:21.670 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[24757]: from='osd.0 [v2:172.21.15.55:6802/42457402,v1:172.21.15.55:6803/42457402]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:28:21.670 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:21 smithi055 ceph-mon[24757]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:28:21.671 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:28:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[32334]: 2024-05-20T22:28:21.263+0000 7ff185370700 -1 osd.0 0 waiting for initial osdmap 2024-05-20T22:28:21.671 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:28:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[32334]: 2024-05-20T22:28:21.265+0000 7ff180508700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:28:22.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:22 smithi138 ceph-mon[25673]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:22.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:22 smithi138 ceph-mon[25673]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:28:22.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:22 smithi138 ceph-mon[25673]: osdmap e7: 1 total, 0 up, 1 in 2024-05-20T22:28:22.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:22 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:22.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:22 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:22.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[24757]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:22.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[24757]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:28:22.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[24757]: osdmap e7: 1 total, 0 up, 1 in 2024-05-20T22:28:22.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:22.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:22.545 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[29498]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:22.545 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[29498]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:28:22.545 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[29498]: osdmap e7: 1 total, 0 up, 1 in 2024-05-20T22:28:22.545 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:22.545 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:22 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:23.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:23 smithi138 ceph-mon[25673]: purged_snaps scrub starts 2024-05-20T22:28:23.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:23 smithi138 ceph-mon[25673]: purged_snaps scrub ok 2024-05-20T22:28:23.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:23.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:23 smithi138 ceph-mon[25673]: osd.0 [v2:172.21.15.55:6802/42457402,v1:172.21.15.55:6803/42457402] boot 2024-05-20T22:28:23.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:23 smithi138 ceph-mon[25673]: osdmap e8: 1 total, 1 up, 1 in 2024-05-20T22:28:23.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:23.727 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[24757]: purged_snaps scrub starts 2024-05-20T22:28:23.728 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[24757]: purged_snaps scrub ok 2024-05-20T22:28:23.728 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:23.728 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[24757]: osd.0 [v2:172.21.15.55:6802/42457402,v1:172.21.15.55:6803/42457402] boot 2024-05-20T22:28:23.728 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[24757]: osdmap e8: 1 total, 1 up, 1 in 2024-05-20T22:28:23.728 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:23.728 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[29498]: purged_snaps scrub starts 2024-05-20T22:28:23.728 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[29498]: purged_snaps scrub ok 2024-05-20T22:28:23.728 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:23.728 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[29498]: osd.0 [v2:172.21.15.55:6802/42457402,v1:172.21.15.55:6803/42457402] boot 2024-05-20T22:28:23.729 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[29498]: osdmap e8: 1 total, 1 up, 1 in 2024-05-20T22:28:23.729 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:28:24.144 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-20T22:28:24.144 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-20T22:28:24.144 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2024-05-20T22:28:24.144 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2024-05-20T22:28:24.144 INFO:teuthology.orchestra.run.smithi055.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0163985 s, 639 MB/s 2024-05-20T22:28:24.144 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 2024-05-20T22:28:24.144 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2024-05-20T22:28:24.419 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:24 smithi055 ceph-mon[24757]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:24.419 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:24 smithi055 ceph-mon[24757]: osdmap e9: 1 total, 1 up, 1 in 2024-05-20T22:28:24.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:24 smithi055 ceph-mon[29498]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:24.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:24 smithi055 ceph-mon[29498]: osdmap e9: 1 total, 1 up, 1 in 2024-05-20T22:28:24.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:24 smithi138 ceph-mon[25673]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:28:24.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:24 smithi138 ceph-mon[25673]: osdmap e9: 1 total, 1 up, 1 in 2024-05-20T22:28:24.853 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi055:vg_nvme/lv_3 2024-05-20T22:28:25.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[24757]: Detected new or changed devices on smithi055 2024-05-20T22:28:25.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:25.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:28:25.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi055 to 16069M 2024-05-20T22:28:25.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:25.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:25.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[29498]: Detected new or changed devices on smithi055 2024-05-20T22:28:25.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:25.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:28:25.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi055 to 16069M 2024-05-20T22:28:25.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:25.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:25 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:25.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:25 smithi138 ceph-mon[25673]: Detected new or changed devices on smithi055 2024-05-20T22:28:25.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:25 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:25.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:25 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:28:25.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:25 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi055 to 16069M 2024-05-20T22:28:25.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:25 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:25.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:25 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:26.376 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:26 smithi055 ceph-mon[24757]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:26.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:26 smithi055 ceph-mon[29498]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:26.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:26 smithi138 ceph-mon[25673]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:27.458 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:27 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:28:27.458 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:27 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:28:27.458 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:27 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:27.458 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:27 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:28:27.458 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:27 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:28:27.459 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:27 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:27.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:27 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:28:27.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:27 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:28:27.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:27 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:28.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:28 smithi055 ceph-mon[24757]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:28.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:28 smithi055 ceph-mon[24757]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:28.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:28 smithi055 ceph-mon[29498]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:28.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:28 smithi055 ceph-mon[29498]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:28.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:28 smithi138 ceph-mon[25673]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:28.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:28 smithi138 ceph-mon[25673]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:29.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:29 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4003097593' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca54976a-fed3-469f-8667-f4047f13689a"}]: dispatch 2024-05-20T22:28:29.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:29 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4003097593' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca54976a-fed3-469f-8667-f4047f13689a"}]': finished 2024-05-20T22:28:29.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:29 smithi055 ceph-mon[24757]: osdmap e10: 2 total, 1 up, 2 in 2024-05-20T22:28:29.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:29 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:29 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/4003097593' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca54976a-fed3-469f-8667-f4047f13689a"}]: dispatch 2024-05-20T22:28:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:29 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/4003097593' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca54976a-fed3-469f-8667-f4047f13689a"}]': finished 2024-05-20T22:28:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:29 smithi055 ceph-mon[29498]: osdmap e10: 2 total, 1 up, 2 in 2024-05-20T22:28:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:29 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:29.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:29 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/4003097593' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca54976a-fed3-469f-8667-f4047f13689a"}]: dispatch 2024-05-20T22:28:29.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:29 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/4003097593' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca54976a-fed3-469f-8667-f4047f13689a"}]': finished 2024-05-20T22:28:29.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:29 smithi138 ceph-mon[25673]: osdmap e10: 2 total, 1 up, 2 in 2024-05-20T22:28:29.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:29 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:30.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:30 smithi055 ceph-mon[29498]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:30.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:30 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/416460825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:28:30.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:30 smithi055 ceph-mon[24757]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:30.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:30 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/416460825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:28:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:30 smithi138 ceph-mon[25673]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:30 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/416460825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:28:31.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:31 smithi055 ceph-mon[24757]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:31.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:31 smithi055 ceph-mon[29498]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:31.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:31 smithi138 ceph-mon[25673]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:34.375 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:34 smithi055 ceph-mon[24757]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:34.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:34 smithi055 ceph-mon[29498]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:34.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:34 smithi138 ceph-mon[25673]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:36.209 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:36 smithi055 ceph-mon[24757]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:36.210 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:28:36.210 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:36.210 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:36 smithi055 ceph-mon[29498]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:36.210 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:28:36.210 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:36.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:36 smithi138 ceph-mon[25673]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:36.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:36 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:28:36.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:36 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:37.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:37 smithi055 ceph-mon[29498]: Deploying daemon osd.1 on smithi055 2024-05-20T22:28:37.410 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:37 smithi055 ceph-mon[24757]: Deploying daemon osd.1 on smithi055 2024-05-20T22:28:37.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:37 smithi138 ceph-mon[25673]: Deploying daemon osd.1 on smithi055 2024-05-20T22:28:38.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:38 smithi055 ceph-mon[24757]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:38.213 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:38 smithi055 ceph-mon[29498]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:38.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:38 smithi138 ceph-mon[25673]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:40.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:40 smithi055 ceph-mon[24757]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:40.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:40 smithi055 ceph-mon[29498]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:40.593 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:40 smithi138 ceph-mon[25673]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:41.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:41 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:41.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:41 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:41.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:41 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:41.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:41 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:41.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:41 smithi138 ceph-mon[25673]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:41.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:41.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:41.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:41.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:41.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[24757]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:41.885 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:41.885 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:41.885 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:41.885 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:41.885 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:41 smithi055 ceph-mon[29498]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:43.673 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 1 on host 'smithi055' 2024-05-20T22:28:44.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:44 smithi055 ceph-mon[24757]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:44.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:44 smithi055 ceph-mon[24757]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:28:44.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:44 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:44.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:44 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:44.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:44 smithi055 ceph-mon[29498]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:44.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:44 smithi055 ceph-mon[29498]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:28:44.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:44 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:44.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:44 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:44.323 DEBUG:teuthology.orchestra.run.smithi055:osd.1> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.1.service 2024-05-20T22:28:44.325 INFO:tasks.cephadm:Deploying osd.2 on smithi055 with /dev/vg_nvme/lv_2... 2024-05-20T22:28:44.325 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-20T22:28:44.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:44 smithi138 ceph-mon[25673]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:44.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:44 smithi138 ceph-mon[25673]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:28:44.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:44 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:44.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:44 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:45.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[24757]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T22:28:45.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[24757]: osdmap e11: 2 total, 1 up, 2 in 2024-05-20T22:28:45.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:45.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[24757]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:28:45.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[24757]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:28:45.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[24757]: osdmap e12: 2 total, 1 up, 2 in 2024-05-20T22:28:45.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:45.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[29498]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T22:28:45.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[29498]: osdmap e11: 2 total, 1 up, 2 in 2024-05-20T22:28:45.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:45.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[29498]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:28:45.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[29498]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:28:45.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[29498]: osdmap e12: 2 total, 1 up, 2 in 2024-05-20T22:28:45.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:45 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:45.489 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:28:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[35281]: 2024-05-20T22:28:45.149+0000 7f9394922700 -1 osd.1 0 waiting for initial osdmap 2024-05-20T22:28:45.489 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:28:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[35281]: 2024-05-20T22:28:45.157+0000 7f938f2b9700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:28:45.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:45 smithi138 ceph-mon[25673]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T22:28:45.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:45 smithi138 ceph-mon[25673]: osdmap e11: 2 total, 1 up, 2 in 2024-05-20T22:28:45.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:45 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:45.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:45 smithi138 ceph-mon[25673]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:28:45.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:45 smithi138 ceph-mon[25673]: from='osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:28:45.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:45 smithi138 ceph-mon[25673]: osdmap e12: 2 total, 1 up, 2 in 2024-05-20T22:28:45.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:45 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:46.319 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:46 smithi055 ceph-mon[24757]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:46.319 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:46.320 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:46 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:46.320 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:46 smithi055 ceph-mon[29498]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:46.320 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:46.320 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:46 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:46.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:46 smithi138 ceph-mon[25673]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:46.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:46 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:46.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:46 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:47.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[24757]: purged_snaps scrub starts 2024-05-20T22:28:47.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[24757]: purged_snaps scrub ok 2024-05-20T22:28:47.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[24757]: osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324] boot 2024-05-20T22:28:47.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[24757]: osdmap e13: 2 total, 2 up, 2 in 2024-05-20T22:28:47.225 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:47.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[29498]: purged_snaps scrub starts 2024-05-20T22:28:47.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[29498]: purged_snaps scrub ok 2024-05-20T22:28:47.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[29498]: osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324] boot 2024-05-20T22:28:47.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[29498]: osdmap e13: 2 total, 2 up, 2 in 2024-05-20T22:28:47.226 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:47.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:47 smithi138 ceph-mon[25673]: purged_snaps scrub starts 2024-05-20T22:28:47.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:47 smithi138 ceph-mon[25673]: purged_snaps scrub ok 2024-05-20T22:28:47.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:47 smithi138 ceph-mon[25673]: osd.1 [v2:172.21.15.55:6810/3429244324,v1:172.21.15.55:6811/3429244324] boot 2024-05-20T22:28:47.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:47 smithi138 ceph-mon[25673]: osdmap e13: 2 total, 2 up, 2 in 2024-05-20T22:28:47.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:48.401 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-20T22:28:48.401 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-20T22:28:48.401 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2024-05-20T22:28:48.402 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2024-05-20T22:28:48.402 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113031 s, 928 MB/s 2024-05-20T22:28:48.402 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2024-05-20T22:28:48.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: pgmap v56: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:48.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: osdmap e14: 2 total, 2 up, 2 in 2024-05-20T22:28:48.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:28:48.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:28:48.403 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.403 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.403 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:48.403 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:48.403 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:48.403 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.403 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: pgmap v56: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:48.403 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: osdmap e14: 2 total, 2 up, 2 in 2024-05-20T22:28:48.403 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:28:48.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:28:48.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:48.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:48.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:48.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: pgmap v56: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:28:48.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: osdmap e14: 2 total, 2 up, 2 in 2024-05-20T22:28:48.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:28:48.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:28:48.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:48.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:48.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:48.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:48.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:28:49.095 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi055:vg_nvme/lv_2 2024-05-20T22:28:49.418 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:49 smithi055 ceph-mon[24757]: Detected new or changed devices on smithi055 2024-05-20T22:28:49.418 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:49 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi055 to 8034M 2024-05-20T22:28:49.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:49 smithi055 ceph-mon[29498]: Detected new or changed devices on smithi055 2024-05-20T22:28:49.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:49 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi055 to 8034M 2024-05-20T22:28:49.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:49 smithi138 ceph-mon[25673]: Detected new or changed devices on smithi055 2024-05-20T22:28:49.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:49 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi055 to 8034M 2024-05-20T22:28:50.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:50 smithi055 ceph-mon[24757]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:50.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:50 smithi055 ceph-mon[29498]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:50.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:50 smithi138 ceph-mon[25673]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:51.456 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:51 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:28:51.456 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:51 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:28:51.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:51 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:51.457 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:51 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:28:51.457 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:51 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:28:51.457 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:51 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:51.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:51 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:28:51.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:51 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:28:51.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:51 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:52.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:52 smithi055 ceph-mon[24757]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:52.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:52 smithi055 ceph-mon[24757]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:52.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:52 smithi055 ceph-mon[29498]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:52.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:52 smithi055 ceph-mon[29498]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:52.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:52 smithi138 ceph-mon[25673]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:52.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:52 smithi138 ceph-mon[25673]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:54.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[24757]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:54.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1196471206' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a10157ad-79cb-4cf7-8422-f68f5f788e44"}]: dispatch 2024-05-20T22:28:54.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a10157ad-79cb-4cf7-8422-f68f5f788e44"}]: dispatch 2024-05-20T22:28:54.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a10157ad-79cb-4cf7-8422-f68f5f788e44"}]': finished 2024-05-20T22:28:54.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[24757]: osdmap e15: 3 total, 2 up, 3 in 2024-05-20T22:28:54.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:28:54.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1183494381' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:28:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[29498]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1196471206' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a10157ad-79cb-4cf7-8422-f68f5f788e44"}]: dispatch 2024-05-20T22:28:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a10157ad-79cb-4cf7-8422-f68f5f788e44"}]: dispatch 2024-05-20T22:28:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a10157ad-79cb-4cf7-8422-f68f5f788e44"}]': finished 2024-05-20T22:28:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[29498]: osdmap e15: 3 total, 2 up, 3 in 2024-05-20T22:28:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:28:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:54 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1183494381' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:28:54.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:54 smithi138 ceph-mon[25673]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:54.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:54 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1196471206' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a10157ad-79cb-4cf7-8422-f68f5f788e44"}]: dispatch 2024-05-20T22:28:54.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:54 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a10157ad-79cb-4cf7-8422-f68f5f788e44"}]: dispatch 2024-05-20T22:28:54.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:54 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a10157ad-79cb-4cf7-8422-f68f5f788e44"}]': finished 2024-05-20T22:28:54.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:54 smithi138 ceph-mon[25673]: osdmap e15: 3 total, 2 up, 3 in 2024-05-20T22:28:54.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:54 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:28:54.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:54 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1183494381' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:28:56.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:56 smithi055 ceph-mon[24757]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:56.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:56 smithi055 ceph-mon[29498]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:56.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:56 smithi138 ceph-mon[25673]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:58.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:28:58 smithi055 ceph-mon[24757]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:58.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:28:58 smithi055 ceph-mon[29498]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:28:58.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:28:58 smithi138 ceph-mon[25673]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:00.495 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:00 smithi055 ceph-mon[24757]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:00.495 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:00 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:29:00.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:00 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:00.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:00 smithi055 ceph-mon[29498]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:00.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:00 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:29:00.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:00 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:00.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:00 smithi138 ceph-mon[25673]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:00.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:00 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:29:00.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:00 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:01.518 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:01 smithi055 ceph-mon[24757]: Deploying daemon osd.2 on smithi055 2024-05-20T22:29:01.518 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:29:01.518 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:29:01.518 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:01 smithi055 ceph-mon[29498]: Deploying daemon osd.2 on smithi055 2024-05-20T22:29:01.519 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:29:01.519 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:29:01.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:01 smithi138 ceph-mon[25673]: Deploying daemon osd.2 on smithi055 2024-05-20T22:29:01.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:29:01.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:29:02.376 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:02 smithi055 ceph-mon[24757]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:02.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:02 smithi055 ceph-mon[29498]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:02.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:02 smithi138 ceph-mon[25673]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:04.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:04 smithi138 ceph-mon[25673]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:04.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:04 smithi055 ceph-mon[24757]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:04.635 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:04 smithi055 ceph-mon[29498]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:06 smithi138 ceph-mon[25673]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:06 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:06 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:06 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:06 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[24757]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[29498]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:06.612 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:06 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:08.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:08 smithi138 ceph-mon[25673]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:08.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:08.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:08.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:08 smithi055 ceph-mon[24757]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:08.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:08.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:08 smithi055 ceph-mon[29498]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:08.787 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 2 on host 'smithi055' 2024-05-20T22:29:09.481 DEBUG:teuthology.orchestra.run.smithi055:osd.2> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.2.service 2024-05-20T22:29:09.484 INFO:tasks.cephadm:Deploying osd.3 on smithi055 with /dev/vg_nvme/lv_1... 2024-05-20T22:29:09.484 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-20T22:29:09.547 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[24757]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T22:29:09.547 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:09.547 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:09.547 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:09.547 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:09.547 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[29498]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T22:29:09.547 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:09.547 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:09.548 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:09.548 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:09 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:09.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:09 smithi138 ceph-mon[25673]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T22:29:09.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:09 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:09.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:09 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:09.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:09 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:09.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:09 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:10.552 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[24757]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:10.552 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[24757]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T22:29:10.552 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[24757]: osdmap e16: 3 total, 2 up, 3 in 2024-05-20T22:29:10.552 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:10.552 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[24757]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:29:10.552 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[29498]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:10.552 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[29498]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T22:29:10.553 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[29498]: osdmap e16: 3 total, 2 up, 3 in 2024-05-20T22:29:10.553 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:10.553 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:10 smithi055 ceph-mon[29498]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:29:10.553 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 22:29:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[38186]: 2024-05-20T22:29:10.306+0000 7f542a8e4700 -1 osd.2 0 waiting for initial osdmap 2024-05-20T22:29:10.553 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 22:29:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[38186]: 2024-05-20T22:29:10.325+0000 7f5424a7a700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:29:10.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:10 smithi138 ceph-mon[25673]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:10.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:10 smithi138 ceph-mon[25673]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T22:29:10.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:10 smithi138 ceph-mon[25673]: osdmap e16: 3 total, 2 up, 3 in 2024-05-20T22:29:10.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:10 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:10.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:10 smithi138 ceph-mon[25673]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:29:11.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:11 smithi138 ceph-mon[25673]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:29:11.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:11 smithi138 ceph-mon[25673]: osdmap e17: 3 total, 2 up, 3 in 2024-05-20T22:29:11.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:11 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:11.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:11 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:11.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:11 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:11.675 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[24757]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:29:11.675 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[24757]: osdmap e17: 3 total, 2 up, 3 in 2024-05-20T22:29:11.675 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:11.675 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:11.675 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:11.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[29498]: from='osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:29:11.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[29498]: osdmap e17: 3 total, 2 up, 3 in 2024-05-20T22:29:11.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:11.676 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:11.676 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:11 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:12.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:12 smithi138 ceph-mon[25673]: purged_snaps scrub starts 2024-05-20T22:29:12.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:12 smithi138 ceph-mon[25673]: purged_snaps scrub ok 2024-05-20T22:29:12.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:12 smithi138 ceph-mon[25673]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:12.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:12 smithi138 ceph-mon[25673]: osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308] boot 2024-05-20T22:29:12.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:12 smithi138 ceph-mon[25673]: osdmap e18: 3 total, 3 up, 3 in 2024-05-20T22:29:12.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:12 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:12.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[29498]: purged_snaps scrub starts 2024-05-20T22:29:12.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[29498]: purged_snaps scrub ok 2024-05-20T22:29:12.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[29498]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:12.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[29498]: osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308] boot 2024-05-20T22:29:12.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[29498]: osdmap e18: 3 total, 3 up, 3 in 2024-05-20T22:29:12.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:12.634 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[24757]: purged_snaps scrub starts 2024-05-20T22:29:12.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[24757]: purged_snaps scrub ok 2024-05-20T22:29:12.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[24757]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:29:12.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[24757]: osd.2 [v2:172.21.15.55:6818/3539256308,v1:172.21.15.55:6819/3539256308] boot 2024-05-20T22:29:12.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[24757]: osdmap e18: 3 total, 3 up, 3 in 2024-05-20T22:29:12.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:12 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:13.410 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-20T22:29:13.410 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-20T22:29:13.410 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2024-05-20T22:29:13.410 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2024-05-20T22:29:13.411 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0109135 s, 961 MB/s 2024-05-20T22:29:13.411 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2024-05-20T22:29:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:13 smithi138 ceph-mon[25673]: osdmap e19: 3 total, 3 up, 3 in 2024-05-20T22:29:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:13 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-20T22:29:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:13 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:13 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:13 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:13 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:13.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:13 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:13.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:13 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:13.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[24757]: osdmap e19: 3 total, 3 up, 3 in 2024-05-20T22:29:13.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[29498]: osdmap e19: 3 total, 3 up, 3 in 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:13.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:13.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:13.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:13.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:13.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:13 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:14.071 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi055:vg_nvme/lv_1 2024-05-20T22:29:14.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[24757]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:14.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[24757]: Detected new or changed devices on smithi055 2024-05-20T22:29:14.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi055 to 5356M 2024-05-20T22:29:14.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-20T22:29:14.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[24757]: osdmap e20: 3 total, 3 up, 3 in 2024-05-20T22:29:14.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T22:29:14.677 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[29498]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:14.678 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[29498]: Detected new or changed devices on smithi055 2024-05-20T22:29:14.678 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi055 to 5356M 2024-05-20T22:29:14.678 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-20T22:29:14.678 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[29498]: osdmap e20: 3 total, 3 up, 3 in 2024-05-20T22:29:14.678 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:14 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T22:29:14.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:14 smithi138 ceph-mon[25673]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:14.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:14 smithi138 ceph-mon[25673]: Detected new or changed devices on smithi055 2024-05-20T22:29:14.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:14 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi055 to 5356M 2024-05-20T22:29:14.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:14 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-20T22:29:14.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:14 smithi138 ceph-mon[25673]: osdmap e20: 3 total, 3 up, 3 in 2024-05-20T22:29:14.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:14 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T22:29:14.987 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:29:14 smithi055 sudo[39470]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:29:14.987 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:29:14 smithi055 sudo[39470]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:14.987 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:29:14 smithi055 sudo[39470]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:15.438 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:15 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-20T22:29:15.438 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:15 smithi055 ceph-mon[29498]: osdmap e21: 3 total, 3 up, 3 in 2024-05-20T22:29:15.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:15 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-20T22:29:15.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:15 smithi055 ceph-mon[24757]: osdmap e21: 3 total, 3 up, 3 in 2024-05-20T22:29:15.439 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:29:15 smithi055 sudo[39476]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:29:15.439 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:29:15 smithi055 sudo[39476]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:15.439 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:29:15 smithi055 sudo[39476]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:15.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:15 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-20T22:29:15.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:15 smithi138 ceph-mon[25673]: osdmap e21: 3 total, 3 up, 3 in 2024-05-20T22:29:15.987 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:29:15 smithi055 sudo[39502]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:29:15.987 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:29:15 smithi055 sudo[39502]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:15.987 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:29:15 smithi055 sudo[39502]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:16.488 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:29:16 smithi055 sudo[39564]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:29:16.488 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:29:16 smithi055 sudo[39564]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:16.488 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:29:16 smithi055 sudo[39564]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:16.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[29498]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:16.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[29498]: osdmap e22: 3 total, 3 up, 3 in 2024-05-20T22:29:16.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[29498]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[24757]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[24757]: osdmap e22: 3 total, 3 up, 3 in 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[24757]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:29:16.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:16 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:16.801 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 22:29:16 smithi055 sudo[39611]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:29:16.801 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 22:29:16 smithi055 sudo[39611]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:16.802 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 22:29:16 smithi055 sudo[39611]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:16.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:16 smithi138 ceph-mon[25673]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:16.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:16 smithi138 ceph-mon[25673]: osdmap e22: 3 total, 3 up, 3 in 2024-05-20T22:29:16.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:16 smithi138 ceph-mon[25673]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-20T22:29:16.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:16 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:16.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:16 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:29:16.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:16 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:29:16.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:16 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:17.487 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 22:29:17 smithi055 sudo[39631]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:29:17.487 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 22:29:17 smithi055 sudo[39631]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:17.487 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 22:29:17 smithi055 sudo[39631]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:17.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:17 smithi055 sudo[39654]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:29:17.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:17 smithi055 sudo[39654]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:17.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:17 smithi055 sudo[39654]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:18.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:18 smithi055 sudo[39664]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:29:18.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:18 smithi055 sudo[39664]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:18.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:18 smithi055 sudo[39664]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:18.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:18 smithi055 sudo[39699]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:29:18.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:18 smithi055 sudo[39699]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:18.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:18 smithi055 sudo[39699]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:19.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:19 smithi055 ceph-mon[24757]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:19.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:19 smithi055 ceph-mon[24757]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:29:19.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:19 smithi055 sudo[39703]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:29:19.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:19 smithi055 sudo[39703]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:19.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:19 smithi055 sudo[39703]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:19.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:19 smithi055 ceph-mon[29498]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:19.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:19 smithi055 ceph-mon[29498]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:29:19.541 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:19 smithi138 sudo[27181]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:29:19.541 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:19 smithi138 sudo[27181]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:19.542 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:19 smithi138 sudo[27181]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:19.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:19 smithi138 sudo[27184]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:29:19.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:19 smithi138 sudo[27184]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:29:19.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:19 smithi138 sudo[27184]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:29:19.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:19 smithi138 ceph-mon[25673]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:19.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:19 smithi138 ceph-mon[25673]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:29:20.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:20.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:29:20.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:29:20.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:29:20.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:29:20.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:29:20.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2491999769' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "39cc9d6e-81f4-4722-a776-662682ced296"}]: dispatch 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2491999769' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "39cc9d6e-81f4-4722-a776-662682ced296"}]': finished 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: osdmap e23: 4 total, 3 up, 4 in 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:20.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:20.388 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2491999769' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "39cc9d6e-81f4-4722-a776-662682ced296"}]: dispatch 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:20.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2491999769' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "39cc9d6e-81f4-4722-a776-662682ced296"}]': finished 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: osdmap e23: 4 total, 3 up, 4 in 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:20.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2491999769' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "39cc9d6e-81f4-4722-a776-662682ced296"}]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:29:20.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:29:20.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:29:20.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2491999769' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "39cc9d6e-81f4-4722-a776-662682ced296"}]': finished 2024-05-20T22:29:20.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: osdmap e23: 4 total, 3 up, 4 in 2024-05-20T22:29:20.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:20.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:21.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:21 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2304857582' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:29:21.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:21 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2304857582' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:29:21.401 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:21 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2304857582' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:29:22.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:22 smithi138 ceph-mon[25673]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:22.455 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:22 smithi055 ceph-mon[24757]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:22.456 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:22 smithi055 ceph-mon[29498]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:24 smithi055 ceph-mon[24757]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-05-20T22:29:24.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:24 smithi055 ceph-mon[29498]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-05-20T22:29:24.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:24 smithi138 ceph-mon[25673]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-05-20T22:29:26.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:26 smithi055 ceph-mon[29498]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-05-20T22:29:26.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:26 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:29:26.457 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:26 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:26.457 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:26 smithi055 ceph-mon[24757]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-05-20T22:29:26.458 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:26 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:29:26.458 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:26 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:26.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:26 smithi138 ceph-mon[25673]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-05-20T22:29:26.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:26 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:29:26.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:26 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:27.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:27 smithi055 ceph-mon[24757]: Deploying daemon osd.3 on smithi055 2024-05-20T22:29:27.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:27 smithi055 ceph-mon[29498]: Deploying daemon osd.3 on smithi055 2024-05-20T22:29:27.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:27 smithi138 ceph-mon[25673]: Deploying daemon osd.3 on smithi055 2024-05-20T22:29:28.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:28 smithi055 ceph-mon[24757]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-05-20T22:29:28.368 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:28 smithi055 ceph-mon[29498]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-05-20T22:29:28.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:28 smithi138 ceph-mon[25673]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-05-20T22:29:30.336 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:30 smithi055 ceph-mon[24757]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-20T22:29:30.337 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:30 smithi055 ceph-mon[29498]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-20T22:29:30.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:30 smithi138 ceph-mon[25673]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-20T22:29:32.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[24757]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 5 op/s 2024-05-20T22:29:32.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:32.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:32.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:32.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:32.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[29498]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 5 op/s 2024-05-20T22:29:32.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:32.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:32.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:32.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:32 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:32.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:32 smithi138 ceph-mon[25673]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 5 op/s 2024-05-20T22:29:32.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:32 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:32.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:32 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:32.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:32 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:32.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:32 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:34.289 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:34 smithi055 ceph-mon[24757]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-05-20T22:29:34.290 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:34 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:34.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:34 smithi055 ceph-mon[29498]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-05-20T22:29:34.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:34 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:34.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:34 smithi138 ceph-mon[25673]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-05-20T22:29:34.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:34 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:34.754 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 3 on host 'smithi055' 2024-05-20T22:29:35.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[24757]: from='osd.3 [v2:172.21.15.55:6826/1051646179,v1:172.21.15.55:6827/1051646179]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:29:35.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[24757]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:29:35.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:35.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:35.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:35.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:35.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:35.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[29498]: from='osd.3 [v2:172.21.15.55:6826/1051646179,v1:172.21.15.55:6827/1051646179]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:29:35.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[29498]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:29:35.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:35.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:35.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:35.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:35.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:35 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:35.426 DEBUG:teuthology.orchestra.run.smithi055:osd.3> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.3.service 2024-05-20T22:29:35.428 INFO:tasks.cephadm:Deploying osd.4 on smithi138 with /dev/vg_nvme/lv_4... 2024-05-20T22:29:35.428 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-20T22:29:35.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:35 smithi138 ceph-mon[25673]: from='osd.3 [v2:172.21.15.55:6826/1051646179,v1:172.21.15.55:6827/1051646179]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:29:35.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:35 smithi138 ceph-mon[25673]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:29:35.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:35 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:35.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:35 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:35.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:35 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:35.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:35 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:35.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:35 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:36.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:36 smithi138 ceph-mon[25673]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:36.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:36 smithi138 ceph-mon[25673]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T22:29:36.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:36 smithi138 ceph-mon[25673]: osdmap e24: 4 total, 3 up, 4 in 2024-05-20T22:29:36.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:36 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:36.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:36 smithi138 ceph-mon[25673]: from='osd.3 [v2:172.21.15.55:6826/1051646179,v1:172.21.15.55:6827/1051646179]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:29:36.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:36 smithi138 ceph-mon[25673]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:29:36.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[29498]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:36.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[29498]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T22:29:36.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[29498]: osdmap e24: 4 total, 3 up, 4 in 2024-05-20T22:29:36.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:36.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[29498]: from='osd.3 [v2:172.21.15.55:6826/1051646179,v1:172.21.15.55:6827/1051646179]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:29:36.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[29498]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:29:36.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[24757]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:36.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[24757]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T22:29:36.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[24757]: osdmap e24: 4 total, 3 up, 4 in 2024-05-20T22:29:36.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:36.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[24757]: from='osd.3 [v2:172.21.15.55:6826/1051646179,v1:172.21.15.55:6827/1051646179]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:29:36.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:36 smithi055 ceph-mon[24757]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:29:36.489 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 22:29:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[41150]: 2024-05-20T22:29:36.238+0000 7f2dbae86700 -1 osd.3 0 waiting for initial osdmap 2024-05-20T22:29:36.489 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 22:29:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[41150]: 2024-05-20T22:29:36.255+0000 7f2db401a700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:29:37.381 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-20T22:29:37.381 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-20T22:29:37.381 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2024-05-20T22:29:37.381 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2024-05-20T22:29:37.381 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110626 s, 948 MB/s 2024-05-20T22:29:37.381 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2024-05-20T22:29:37.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:37 smithi138 ceph-mon[25673]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:29:37.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:37 smithi138 ceph-mon[25673]: osdmap e25: 4 total, 3 up, 4 in 2024-05-20T22:29:37.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:37 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:37.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:37 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:37.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:37 smithi055 ceph-mon[24757]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:29:37.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:37 smithi055 ceph-mon[24757]: osdmap e25: 4 total, 3 up, 4 in 2024-05-20T22:29:37.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:37 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:37.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:37 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:37.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:37 smithi055 ceph-mon[29498]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-05-20T22:29:37.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:37 smithi055 ceph-mon[29498]: osdmap e25: 4 total, 3 up, 4 in 2024-05-20T22:29:37.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:37 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:37.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:37 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:37.949 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi138:vg_nvme/lv_4 2024-05-20T22:29:38.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:38 smithi138 ceph-mon[25673]: purged_snaps scrub starts 2024-05-20T22:29:38.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:38 smithi138 ceph-mon[25673]: purged_snaps scrub ok 2024-05-20T22:29:38.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:38 smithi138 ceph-mon[25673]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:38.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:38 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:38.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:38 smithi138 ceph-mon[25673]: osd.3 [v2:172.21.15.55:6826/1051646179,v1:172.21.15.55:6827/1051646179] boot 2024-05-20T22:29:38.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:38 smithi138 ceph-mon[25673]: osdmap e26: 4 total, 4 up, 4 in 2024-05-20T22:29:38.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:38 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[24757]: purged_snaps scrub starts 2024-05-20T22:29:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[24757]: purged_snaps scrub ok 2024-05-20T22:29:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[24757]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[24757]: osd.3 [v2:172.21.15.55:6826/1051646179,v1:172.21.15.55:6827/1051646179] boot 2024-05-20T22:29:38.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[24757]: osdmap e26: 4 total, 4 up, 4 in 2024-05-20T22:29:38.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:38.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[29498]: purged_snaps scrub starts 2024-05-20T22:29:38.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[29498]: purged_snaps scrub ok 2024-05-20T22:29:38.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[29498]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:29:38.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:38.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[29498]: osd.3 [v2:172.21.15.55:6826/1051646179,v1:172.21.15.55:6827/1051646179] boot 2024-05-20T22:29:38.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[29498]: osdmap e26: 4 total, 4 up, 4 in 2024-05-20T22:29:38.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:38 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:39.471 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: osdmap e27: 4 total, 4 up, 4 in 2024-05-20T22:29:39.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: Detected new or changed devices on smithi055 2024-05-20T22:29:39.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:39.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi055 to 4017M 2024-05-20T22:29:39.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:39.472 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:39.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: osdmap e27: 4 total, 4 up, 4 in 2024-05-20T22:29:39.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: Detected new or changed devices on smithi055 2024-05-20T22:29:39.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:39.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi055 to 4017M 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: osdmap e27: 4 total, 4 up, 4 in 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: Detected new or changed devices on smithi055 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:29:39.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi055 to 4017M 2024-05-20T22:29:39.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:39.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:40.502 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:40 smithi138 ceph-mon[25673]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:40.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:40 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:29:40.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:40 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:29:40.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:40 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:40.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:40 smithi055 ceph-mon[24757]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:40.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:40 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:29:40.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:40 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:29:40.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:40 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:40.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:40 smithi055 ceph-mon[29498]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:40.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:40 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:29:40.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:40 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:29:40.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:40 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:41.598 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:41 smithi138 ceph-mon[25673]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:41.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:41 smithi055 ceph-mon[24757]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:41.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:41 smithi055 ceph-mon[29498]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:42.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:42 smithi138 ceph-mon[25673]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:42.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:42 smithi138 ceph-mon[25673]: from='client.? 172.21.15.138:0/1439967937' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "100258f9-7e57-450f-b734-60a4ef197a15"}]: dispatch 2024-05-20T22:29:42.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:42 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "100258f9-7e57-450f-b734-60a4ef197a15"}]: dispatch 2024-05-20T22:29:42.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:42 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "100258f9-7e57-450f-b734-60a4ef197a15"}]': finished 2024-05-20T22:29:42.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:42 smithi138 ceph-mon[25673]: osdmap e28: 5 total, 4 up, 5 in 2024-05-20T22:29:42.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:42 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[24757]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/1439967937' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "100258f9-7e57-450f-b734-60a4ef197a15"}]: dispatch 2024-05-20T22:29:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "100258f9-7e57-450f-b734-60a4ef197a15"}]: dispatch 2024-05-20T22:29:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "100258f9-7e57-450f-b734-60a4ef197a15"}]': finished 2024-05-20T22:29:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[24757]: osdmap e28: 5 total, 4 up, 5 in 2024-05-20T22:29:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[29498]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/1439967937' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "100258f9-7e57-450f-b734-60a4ef197a15"}]: dispatch 2024-05-20T22:29:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "100258f9-7e57-450f-b734-60a4ef197a15"}]: dispatch 2024-05-20T22:29:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "100258f9-7e57-450f-b734-60a4ef197a15"}]': finished 2024-05-20T22:29:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[29498]: osdmap e28: 5 total, 4 up, 5 in 2024-05-20T22:29:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:42 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:43.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:43 smithi138 ceph-mon[25673]: from='client.? 172.21.15.138:0/3523157796' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:29:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:43 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/3523157796' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:29:43.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:43 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/3523157796' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:29:44.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:44 smithi138 ceph-mon[25673]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:44.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:44 smithi055 ceph-mon[24757]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:44.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:44 smithi055 ceph-mon[29498]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:46.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:46 smithi138 ceph-mon[25673]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:46.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:46 smithi055 ceph-mon[24757]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:46.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:46 smithi055 ceph-mon[29498]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:48.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:48 smithi138 ceph-mon[25673]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:48.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:29:48.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:48.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:48 smithi055 ceph-mon[24757]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:48.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:29:48.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:48.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:48 smithi055 ceph-mon[29498]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:48.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:29:48.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:49.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:49 smithi138 ceph-mon[25673]: Deploying daemon osd.4 on smithi138 2024-05-20T22:29:49.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:49 smithi055 ceph-mon[24757]: Deploying daemon osd.4 on smithi138 2024-05-20T22:29:49.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:49 smithi055 ceph-mon[29498]: Deploying daemon osd.4 on smithi138 2024-05-20T22:29:50.635 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:50 smithi138 ceph-mon[25673]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:50.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:50 smithi055 ceph-mon[24757]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:50.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:50 smithi055 ceph-mon[29498]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:52.429 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:52 smithi138 ceph-mon[25673]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:52.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:52 smithi055 ceph-mon[29498]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:52.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:52 smithi055 ceph-mon[24757]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:54.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[24757]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:54.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:54.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:54.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:54.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:54.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[29498]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:54.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:54.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:54.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:54.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:53 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:53 smithi138 ceph-mon[25673]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:53 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:53 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:53 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:53 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:56.043 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 4 on host 'smithi138' 2024-05-20T22:29:56.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:56 smithi138 ceph-mon[25673]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:56.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:56 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:56.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:56 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:56.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:56 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:56.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:56 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:56.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:56 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:56.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:56 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:56.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:56 smithi138 ceph-mon[25673]: from='osd.4 [v2:172.21.15.138:6800/2210516721,v1:172.21.15.138:6801/2210516721]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:29:56.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:56 smithi138 ceph-mon[25673]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:29:56.609 DEBUG:teuthology.orchestra.run.smithi138:osd.4> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.4.service 2024-05-20T22:29:56.611 INFO:tasks.cephadm:Deploying osd.5 on smithi138 with /dev/vg_nvme/lv_3... 2024-05-20T22:29:56.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-20T22:29:56.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[24757]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:56.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:56.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:56.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:56.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:56.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:56.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[24757]: from='osd.4 [v2:172.21.15.138:6800/2210516721,v1:172.21.15.138:6801/2210516721]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[24757]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[29498]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:29:56.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[29498]: from='osd.4 [v2:172.21.15.138:6800/2210516721,v1:172.21.15.138:6801/2210516721]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:29:56.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:56 smithi055 ceph-mon[29498]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:29:58.162 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:58 smithi138 ceph-mon[25673]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:58.162 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:58 smithi138 ceph-mon[25673]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T22:29:58.162 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:58 smithi138 ceph-mon[25673]: osdmap e29: 5 total, 4 up, 5 in 2024-05-20T22:29:58.162 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:58 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:58.162 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:58 smithi138 ceph-mon[25673]: from='osd.4 [v2:172.21.15.138:6800/2210516721,v1:172.21.15.138:6801/2210516721]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:29:58.162 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:58 smithi138 ceph-mon[25673]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:29:58.162 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 22:29:58 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[29020]: 2024-05-20T22:29:58.055+0000 7fea3103e700 -1 osd.4 0 waiting for initial osdmap 2024-05-20T22:29:58.162 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 22:29:58 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[29020]: 2024-05-20T22:29:58.059+0000 7fea2b1d4700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:29:58.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[24757]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:58.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[24757]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T22:29:58.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[24757]: osdmap e29: 5 total, 4 up, 5 in 2024-05-20T22:29:58.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:58.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[24757]: from='osd.4 [v2:172.21.15.138:6800/2210516721,v1:172.21.15.138:6801/2210516721]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:29:58.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[24757]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:29:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[29498]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:29:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[29498]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T22:29:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[29498]: osdmap e29: 5 total, 4 up, 5 in 2024-05-20T22:29:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[29498]: from='osd.4 [v2:172.21.15.138:6800/2210516721,v1:172.21.15.138:6801/2210516721]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:29:58.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:58 smithi055 ceph-mon[29498]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:29:59.208 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:59 smithi138 ceph-mon[25673]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:29:59.208 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:59 smithi138 ceph-mon[25673]: osdmap e30: 5 total, 4 up, 5 in 2024-05-20T22:29:59.208 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:59 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:59.208 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:59 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:59.208 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:59 smithi138 ceph-mon[25673]: osd.4 [v2:172.21.15.138:6800/2210516721,v1:172.21.15.138:6801/2210516721] boot 2024-05-20T22:29:59.208 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:59 smithi138 ceph-mon[25673]: osdmap e31: 5 total, 5 up, 5 in 2024-05-20T22:29:59.208 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:29:59 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:59.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[24757]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:29:59.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[24757]: osdmap e30: 5 total, 4 up, 5 in 2024-05-20T22:29:59.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:59.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[24757]: osd.4 [v2:172.21.15.138:6800/2210516721,v1:172.21.15.138:6801/2210516721] boot 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[24757]: osdmap e31: 5 total, 5 up, 5 in 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[29498]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[29498]: osdmap e30: 5 total, 4 up, 5 in 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[29498]: osd.4 [v2:172.21.15.138:6800/2210516721,v1:172.21.15.138:6801/2210516721] boot 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[29498]: osdmap e31: 5 total, 5 up, 5 in 2024-05-20T22:29:59.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:29:59 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:30:00.106 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-20T22:30:00.106 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-20T22:30:00.107 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2024-05-20T22:30:00.107 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2024-05-20T22:30:00.107 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0166929 s, 628 MB/s 2024-05-20T22:30:00.107 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2024-05-20T22:30:00.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:00 smithi138 ceph-mon[25673]: purged_snaps scrub starts 2024-05-20T22:30:00.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:00 smithi138 ceph-mon[25673]: purged_snaps scrub ok 2024-05-20T22:30:00.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:00 smithi138 ceph-mon[25673]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:30:00.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:00 smithi138 ceph-mon[25673]: overall HEALTH_OK 2024-05-20T22:30:00.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:00 smithi138 ceph-mon[25673]: osdmap e32: 5 total, 5 up, 5 in 2024-05-20T22:30:00.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[24757]: purged_snaps scrub starts 2024-05-20T22:30:00.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[24757]: purged_snaps scrub ok 2024-05-20T22:30:00.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[24757]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:30:00.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[24757]: overall HEALTH_OK 2024-05-20T22:30:00.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[24757]: osdmap e32: 5 total, 5 up, 5 in 2024-05-20T22:30:00.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[29498]: purged_snaps scrub starts 2024-05-20T22:30:00.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[29498]: purged_snaps scrub ok 2024-05-20T22:30:00.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[29498]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:30:00.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[29498]: overall HEALTH_OK 2024-05-20T22:30:00.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:00 smithi055 ceph-mon[29498]: osdmap e32: 5 total, 5 up, 5 in 2024-05-20T22:30:00.660 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi138:vg_nvme/lv_3 2024-05-20T22:30:01.234 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:30:01.234 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:30:01.234 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:01.234 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:01.234 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:01.234 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:01.234 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:01 smithi138 ceph-mon[25673]: osdmap e33: 5 total, 5 up, 5 in 2024-05-20T22:30:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:30:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:30:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:01.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[24757]: osdmap e33: 5 total, 5 up, 5 in 2024-05-20T22:30:01.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:30:01.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:30:01.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:01.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:01.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:01.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:01.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:01 smithi055 ceph-mon[29498]: osdmap e33: 5 total, 5 up, 5 in 2024-05-20T22:30:02.241 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:02 smithi138 ceph-mon[25673]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:02.242 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:02 smithi138 ceph-mon[25673]: Detected new or changed devices on smithi138 2024-05-20T22:30:02.242 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:02 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi138 to 17088M 2024-05-20T22:30:02.337 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:02 smithi055 ceph-mon[24757]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:02.337 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:02 smithi055 ceph-mon[24757]: Detected new or changed devices on smithi138 2024-05-20T22:30:02.337 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:02 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi138 to 17088M 2024-05-20T22:30:02.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:02 smithi055 ceph-mon[29498]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:02.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:02 smithi055 ceph-mon[29498]: Detected new or changed devices on smithi138 2024-05-20T22:30:02.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:02 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi138 to 17088M 2024-05-20T22:30:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:03 smithi055 ceph-mon[29498]: osdmap e34: 5 total, 5 up, 5 in 2024-05-20T22:30:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:03 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:30:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:03 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:30:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:03 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:03 smithi055 ceph-mon[24757]: osdmap e34: 5 total, 5 up, 5 in 2024-05-20T22:30:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:03 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:30:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:03 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:30:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:03 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:03.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:03 smithi138 ceph-mon[25673]: osdmap e34: 5 total, 5 up, 5 in 2024-05-20T22:30:03.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:03 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:30:03.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:03 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:30:03.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:03 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:04.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:04 smithi138 ceph-mon[25673]: from='client.34229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:04.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:04 smithi138 ceph-mon[25673]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:30:04.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:04 smithi138 ceph-mon[25673]: osdmap e35: 5 total, 5 up, 5 in 2024-05-20T22:30:04.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:04 smithi055 ceph-mon[24757]: from='client.34229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:04.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:04 smithi055 ceph-mon[24757]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:30:04.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:04 smithi055 ceph-mon[24757]: osdmap e35: 5 total, 5 up, 5 in 2024-05-20T22:30:04.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:04 smithi055 ceph-mon[29498]: from='client.34229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:04.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:04 smithi055 ceph-mon[29498]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:30:04.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:04 smithi055 ceph-mon[29498]: osdmap e35: 5 total, 5 up, 5 in 2024-05-20T22:30:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:05 smithi138 ceph-mon[25673]: from='client.? 172.21.15.138:0/1295714224' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db0c9b42-af8d-4903-9368-17f97b5bf09f"}]: dispatch 2024-05-20T22:30:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:05 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db0c9b42-af8d-4903-9368-17f97b5bf09f"}]: dispatch 2024-05-20T22:30:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:05 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "db0c9b42-af8d-4903-9368-17f97b5bf09f"}]': finished 2024-05-20T22:30:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:05 smithi138 ceph-mon[25673]: osdmap e36: 6 total, 5 up, 6 in 2024-05-20T22:30:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:05 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:05.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/1295714224' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db0c9b42-af8d-4903-9368-17f97b5bf09f"}]: dispatch 2024-05-20T22:30:05.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db0c9b42-af8d-4903-9368-17f97b5bf09f"}]: dispatch 2024-05-20T22:30:05.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "db0c9b42-af8d-4903-9368-17f97b5bf09f"}]': finished 2024-05-20T22:30:05.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[24757]: osdmap e36: 6 total, 5 up, 6 in 2024-05-20T22:30:05.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:05.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/1295714224' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db0c9b42-af8d-4903-9368-17f97b5bf09f"}]: dispatch 2024-05-20T22:30:05.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db0c9b42-af8d-4903-9368-17f97b5bf09f"}]: dispatch 2024-05-20T22:30:05.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "db0c9b42-af8d-4903-9368-17f97b5bf09f"}]': finished 2024-05-20T22:30:05.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[29498]: osdmap e36: 6 total, 5 up, 6 in 2024-05-20T22:30:05.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:05 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:06.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:06 smithi138 ceph-mon[25673]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:30:06.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:06 smithi138 ceph-mon[25673]: from='client.? 172.21.15.138:0/475601591' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:30:06.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:06 smithi055 ceph-mon[24757]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:30:06.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:06 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/475601591' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:30:06.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:06 smithi055 ceph-mon[29498]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:30:06.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:06 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/475601591' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:30:08.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:08 smithi138 ceph-mon[25673]: pgmap v119: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-05-20T22:30:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:08 smithi055 ceph-mon[24757]: pgmap v119: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-05-20T22:30:08.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:08 smithi055 ceph-mon[29498]: pgmap v119: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-05-20T22:30:10.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:10 smithi138 ceph-mon[25673]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-05-20T22:30:10.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:10 smithi055 ceph-mon[24757]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-05-20T22:30:10.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:10 smithi055 ceph-mon[29498]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-05-20T22:30:11.286 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:11 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:30:11.286 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:11 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:11.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:11 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:30:11.490 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:11 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:11.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:11 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:30:11.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:11 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:12.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:12 smithi138 ceph-mon[25673]: Deploying daemon osd.5 on smithi138 2024-05-20T22:30:12.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:12 smithi138 ceph-mon[25673]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:12 smithi055 ceph-mon[24757]: Deploying daemon osd.5 on smithi138 2024-05-20T22:30:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:12 smithi055 ceph-mon[24757]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:12.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:12 smithi055 ceph-mon[29498]: Deploying daemon osd.5 on smithi138 2024-05-20T22:30:12.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:12 smithi055 ceph-mon[29498]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:14.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:14 smithi138 ceph-mon[25673]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:14.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:14 smithi055 ceph-mon[24757]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:14.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:14 smithi055 ceph-mon[29498]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:16.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[29498]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:16.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:16.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:16.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:16.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:16.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[24757]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:16.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:16.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:16.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:16.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:16 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:16.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:16 smithi138 ceph-mon[25673]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:16.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:16 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:16.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:16 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:16.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:16 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:16.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:16 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:18.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:18 smithi138 ceph-mon[25673]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:18.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:18 smithi055 ceph-mon[29498]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:18.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:18 smithi055 ceph-mon[24757]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:18.500 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 5 on host 'smithi138' 2024-05-20T22:30:19.140 DEBUG:teuthology.orchestra.run.smithi138:osd.5> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.5.service 2024-05-20T22:30:19.142 INFO:tasks.cephadm:Deploying osd.6 on smithi138 with /dev/vg_nvme/lv_2... 2024-05-20T22:30:19.142 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-20T22:30:19.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:19 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:19.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:19 smithi138 ceph-mon[25673]: from='osd.5 [v2:172.21.15.138:6808/2265959642,v1:172.21.15.138:6809/2265959642]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:30:19.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:19 smithi138 ceph-mon[25673]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:30:19.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:19 smithi138 ceph-mon[25673]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:19.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:19 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:19.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:19.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[29498]: from='osd.5 [v2:172.21.15.138:6808/2265959642,v1:172.21.15.138:6809/2265959642]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:30:19.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[29498]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:30:19.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[29498]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:19.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:19.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:19.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[24757]: from='osd.5 [v2:172.21.15.138:6808/2265959642,v1:172.21.15.138:6809/2265959642]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:30:19.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[24757]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:30:19.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[24757]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:19.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:19 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:20.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:20 smithi138 ceph-mon[25673]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T22:30:20.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:20 smithi138 ceph-mon[25673]: osdmap e37: 6 total, 5 up, 6 in 2024-05-20T22:30:20.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:20 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:20.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:20 smithi138 ceph-mon[25673]: from='osd.5 [v2:172.21.15.138:6808/2265959642,v1:172.21.15.138:6809/2265959642]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:20.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:20 smithi138 ceph-mon[25673]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:20.861 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 22:30:20 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[31956]: 2024-05-20T22:30:20.523+0000 7f44c7282700 -1 osd.5 0 waiting for initial osdmap 2024-05-20T22:30:20.861 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 22:30:20 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[31956]: 2024-05-20T22:30:20.530+0000 7f44c1c19700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:30:20.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[29498]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T22:30:20.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[29498]: osdmap e37: 6 total, 5 up, 6 in 2024-05-20T22:30:20.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:20.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[29498]: from='osd.5 [v2:172.21.15.138:6808/2265959642,v1:172.21.15.138:6809/2265959642]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:20.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[29498]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:20.890 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[24757]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T22:30:20.890 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[24757]: osdmap e37: 6 total, 5 up, 6 in 2024-05-20T22:30:20.890 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:20.890 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[24757]: from='osd.5 [v2:172.21.15.138:6808/2265959642,v1:172.21.15.138:6809/2265959642]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:20.890 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:20 smithi055 ceph-mon[24757]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:21.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:21 smithi138 ceph-mon[25673]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:30:21.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:21 smithi138 ceph-mon[25673]: osdmap e38: 6 total, 5 up, 6 in 2024-05-20T22:30:21.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:21 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:21.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:21 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:21.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:21 smithi138 ceph-mon[25673]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:21.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:21 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:21.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[24757]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:30:21.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[24757]: osdmap e38: 6 total, 5 up, 6 in 2024-05-20T22:30:21.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:21.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:21.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[24757]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:21.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:21.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[29498]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:30:21.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[29498]: osdmap e38: 6 total, 5 up, 6 in 2024-05-20T22:30:21.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:21.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:21.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[29498]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:30:21.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:21 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:22.830 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:22 smithi138 ceph-mon[25673]: purged_snaps scrub starts 2024-05-20T22:30:22.830 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:22 smithi138 ceph-mon[25673]: purged_snaps scrub ok 2024-05-20T22:30:22.830 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:22 smithi138 ceph-mon[25673]: osd.5 [v2:172.21.15.138:6808/2265959642,v1:172.21.15.138:6809/2265959642] boot 2024-05-20T22:30:22.830 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:22 smithi138 ceph-mon[25673]: osdmap e39: 6 total, 6 up, 6 in 2024-05-20T22:30:22.830 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:22 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:22.830 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:22 smithi138 ceph-mon[25673]: osdmap e40: 6 total, 6 up, 6 in 2024-05-20T22:30:22.831 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-20T22:30:22.831 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-20T22:30:22.831 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2024-05-20T22:30:22.831 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2024-05-20T22:30:22.831 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0150377 s, 697 MB/s 2024-05-20T22:30:22.831 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2024-05-20T22:30:22.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[29498]: purged_snaps scrub starts 2024-05-20T22:30:22.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[29498]: purged_snaps scrub ok 2024-05-20T22:30:22.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[29498]: osd.5 [v2:172.21.15.138:6808/2265959642,v1:172.21.15.138:6809/2265959642] boot 2024-05-20T22:30:22.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[29498]: osdmap e39: 6 total, 6 up, 6 in 2024-05-20T22:30:22.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:22.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[29498]: osdmap e40: 6 total, 6 up, 6 in 2024-05-20T22:30:22.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[24757]: purged_snaps scrub starts 2024-05-20T22:30:22.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[24757]: purged_snaps scrub ok 2024-05-20T22:30:22.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[24757]: osd.5 [v2:172.21.15.138:6808/2265959642,v1:172.21.15.138:6809/2265959642] boot 2024-05-20T22:30:22.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[24757]: osdmap e39: 6 total, 6 up, 6 in 2024-05-20T22:30:22.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:30:22.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:22 smithi055 ceph-mon[24757]: osdmap e40: 6 total, 6 up, 6 in 2024-05-20T22:30:23.483 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi138:vg_nvme/lv_2 2024-05-20T22:30:24.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:24.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: Detected new or changed devices on smithi138 2024-05-20T22:30:24.046 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi138 to 8544M 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.047 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:23 smithi138 ceph-mon[25673]: osdmap e41: 6 total, 6 up, 6 in 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: Detected new or changed devices on smithi138 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi138 to 8544M 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:24.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[29498]: osdmap e41: 6 total, 6 up, 6 in 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: Detected new or changed devices on smithi138 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi138 to 8544M 2024-05-20T22:30:24.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:24.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:24.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:24.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:24.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:23 smithi055 ceph-mon[24757]: osdmap e41: 6 total, 6 up, 6 in 2024-05-20T22:30:25.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:25 smithi138 ceph-mon[25673]: osdmap e42: 6 total, 6 up, 6 in 2024-05-20T22:30:25.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:25 smithi055 ceph-mon[24757]: osdmap e42: 6 total, 6 up, 6 in 2024-05-20T22:30:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:25 smithi055 ceph-mon[29498]: osdmap e42: 6 total, 6 up, 6 in 2024-05-20T22:30:26.467 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:26 smithi138 ceph-mon[25673]: pgmap v134: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:26.467 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:26 smithi138 ceph-mon[25673]: osdmap e43: 6 total, 6 up, 6 in 2024-05-20T22:30:26.467 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:26 smithi138 ceph-mon[25673]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:26.467 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:26 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:30:26.467 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:26 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:30:26.467 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:26 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:26.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[24757]: pgmap v134: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:26.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[24757]: osdmap e43: 6 total, 6 up, 6 in 2024-05-20T22:30:26.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[24757]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:26.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:30:26.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:30:26.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:26.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[29498]: pgmap v134: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:26.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[29498]: osdmap e43: 6 total, 6 up, 6 in 2024-05-20T22:30:26.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[29498]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:26.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:30:26.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:30:26.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:26 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[24757]: pgmap v136: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/256144035' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "13c8d3de-41af-430a-912c-cdaab84f62d0"}]: dispatch 2024-05-20T22:30:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "13c8d3de-41af-430a-912c-cdaab84f62d0"}]: dispatch 2024-05-20T22:30:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "13c8d3de-41af-430a-912c-cdaab84f62d0"}]': finished 2024-05-20T22:30:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[24757]: osdmap e44: 7 total, 6 up, 7 in 2024-05-20T22:30:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/3904680372' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:30:28.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[29498]: pgmap v136: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:28.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/256144035' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "13c8d3de-41af-430a-912c-cdaab84f62d0"}]: dispatch 2024-05-20T22:30:28.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "13c8d3de-41af-430a-912c-cdaab84f62d0"}]: dispatch 2024-05-20T22:30:28.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "13c8d3de-41af-430a-912c-cdaab84f62d0"}]': finished 2024-05-20T22:30:28.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[29498]: osdmap e44: 7 total, 6 up, 7 in 2024-05-20T22:30:28.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:28.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:28 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/3904680372' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:30:28.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:28 smithi138 ceph-mon[25673]: pgmap v136: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:28.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:28 smithi138 ceph-mon[25673]: from='client.? 172.21.15.138:0/256144035' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "13c8d3de-41af-430a-912c-cdaab84f62d0"}]: dispatch 2024-05-20T22:30:28.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:28 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "13c8d3de-41af-430a-912c-cdaab84f62d0"}]: dispatch 2024-05-20T22:30:28.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:28 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "13c8d3de-41af-430a-912c-cdaab84f62d0"}]': finished 2024-05-20T22:30:28.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:28 smithi138 ceph-mon[25673]: osdmap e44: 7 total, 6 up, 7 in 2024-05-20T22:30:28.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:28 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:28.611 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:28 smithi138 ceph-mon[25673]: from='client.? 172.21.15.138:0/3904680372' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:30:30.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:30 smithi055 ceph-mon[24757]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:30.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:30 smithi055 ceph-mon[29498]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:30.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:30 smithi138 ceph-mon[25673]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:32.283 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:32 smithi138 ceph-mon[25673]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:32.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:32 smithi055 ceph-mon[29498]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:32.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:32 smithi055 ceph-mon[24757]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:34 smithi138 ceph-mon[25673]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:34 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:30:34.442 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:34 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:34.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:34 smithi055 ceph-mon[24757]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:34.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:34 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:30:34.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:34 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:34.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:34 smithi055 ceph-mon[29498]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:34.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:34 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:30:34.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:34 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:35.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:35 smithi055 ceph-mon[24757]: Deploying daemon osd.6 on smithi138 2024-05-20T22:30:35.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:35 smithi055 ceph-mon[29498]: Deploying daemon osd.6 on smithi138 2024-05-20T22:30:35.518 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:35 smithi138 ceph-mon[25673]: Deploying daemon osd.6 on smithi138 2024-05-20T22:30:36.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:36 smithi138 ceph-mon[25673]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:36.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:36 smithi055 ceph-mon[24757]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:36.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:36 smithi055 ceph-mon[29498]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:38.349 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:38 smithi138 ceph-mon[25673]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:38.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:38 smithi055 ceph-mon[24757]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:38.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:38 smithi055 ceph-mon[29498]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:39.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:39.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:39.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:39.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:39.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[24757]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:39.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:39 smithi055 ceph-mon[29498]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:39.769 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:39.770 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:39.770 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:39.770 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:39 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:39.770 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:39 smithi138 ceph-mon[25673]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:41.294 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 6 on host 'smithi138' 2024-05-20T22:30:41.932 DEBUG:teuthology.orchestra.run.smithi138:osd.6> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.6.service 2024-05-20T22:30:41.934 INFO:tasks.cephadm:Deploying osd.7 on smithi138 with /dev/vg_nvme/lv_1... 2024-05-20T22:30:41.934 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-20T22:30:42.602 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:42 smithi138 ceph-mon[25673]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:42.602 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:42 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:42.602 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:42 smithi138 ceph-mon[25673]: from='osd.6 [v2:172.21.15.138:6816/3675925959,v1:172.21.15.138:6817/3675925959]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:30:42.603 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:42 smithi138 ceph-mon[25673]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:30:42.603 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:42 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[24757]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[24757]: from='osd.6 [v2:172.21.15.138:6816/3675925959,v1:172.21.15.138:6817/3675925959]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:30:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[24757]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:30:42.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[29498]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[29498]: from='osd.6 [v2:172.21.15.138:6816/3675925959,v1:172.21.15.138:6817/3675925959]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:30:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[29498]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:30:42.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:42 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:43.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:43 smithi138 ceph-mon[25673]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T22:30:43.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:43 smithi138 ceph-mon[25673]: osdmap e45: 7 total, 6 up, 7 in 2024-05-20T22:30:43.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:43 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:43.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:43 smithi138 ceph-mon[25673]: from='osd.6 [v2:172.21.15.138:6816/3675925959,v1:172.21.15.138:6817/3675925959]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:43.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:43 smithi138 ceph-mon[25673]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:43.585 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 22:30:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[34833]: 2024-05-20T22:30:43.330+0000 7f2b7d6e1700 -1 osd.6 0 waiting for initial osdmap 2024-05-20T22:30:43.585 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 22:30:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[34833]: 2024-05-20T22:30:43.336+0000 7f2b77877700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:30:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[24757]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T22:30:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[24757]: osdmap e45: 7 total, 6 up, 7 in 2024-05-20T22:30:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[24757]: from='osd.6 [v2:172.21.15.138:6816/3675925959,v1:172.21.15.138:6817/3675925959]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[24757]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:43.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[29498]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T22:30:43.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[29498]: osdmap e45: 7 total, 6 up, 7 in 2024-05-20T22:30:43.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:43.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[29498]: from='osd.6 [v2:172.21.15.138:6816/3675925959,v1:172.21.15.138:6817/3675925959]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:43.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:43 smithi055 ceph-mon[29498]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:30:44.411 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-20T22:30:44.411 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-20T22:30:44.411 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2024-05-20T22:30:44.412 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2024-05-20T22:30:44.412 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0120676 s, 869 MB/s 2024-05-20T22:30:44.412 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2024-05-20T22:30:44.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:44 smithi138 ceph-mon[25673]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:44.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:44 smithi138 ceph-mon[25673]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:30:44.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:44 smithi138 ceph-mon[25673]: osdmap e46: 7 total, 6 up, 7 in 2024-05-20T22:30:44.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:44 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:44.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:44 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:44.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[24757]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:44.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[24757]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:30:44.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[24757]: osdmap e46: 7 total, 6 up, 7 in 2024-05-20T22:30:44.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:44.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:44.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[29498]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:30:44.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[29498]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:30:44.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[29498]: osdmap e46: 7 total, 6 up, 7 in 2024-05-20T22:30:44.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:44.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:44 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:45.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi138:vg_nvme/lv_1 2024-05-20T22:30:45.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:45 smithi138 ceph-mon[25673]: purged_snaps scrub starts 2024-05-20T22:30:45.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:45 smithi138 ceph-mon[25673]: purged_snaps scrub ok 2024-05-20T22:30:45.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:45 smithi138 ceph-mon[25673]: osd.6 [v2:172.21.15.138:6816/3675925959,v1:172.21.15.138:6817/3675925959] boot 2024-05-20T22:30:45.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:45 smithi138 ceph-mon[25673]: osdmap e47: 7 total, 7 up, 7 in 2024-05-20T22:30:45.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:45 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[24757]: purged_snaps scrub starts 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[24757]: purged_snaps scrub ok 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[24757]: osd.6 [v2:172.21.15.138:6816/3675925959,v1:172.21.15.138:6817/3675925959] boot 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[24757]: osdmap e47: 7 total, 7 up, 7 in 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[29498]: purged_snaps scrub starts 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[29498]: purged_snaps scrub ok 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[29498]: osd.6 [v2:172.21.15.138:6816/3675925959,v1:172.21.15.138:6817/3675925959] boot 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[29498]: osdmap e47: 7 total, 7 up, 7 in 2024-05-20T22:30:45.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:45 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:30:46.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:46 smithi138 ceph-mon[25673]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:30:46.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:46 smithi138 ceph-mon[25673]: osdmap e48: 7 total, 7 up, 7 in 2024-05-20T22:30:46.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:46 smithi055 ceph-mon[24757]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:30:46.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:46 smithi055 ceph-mon[24757]: osdmap e48: 7 total, 7 up, 7 in 2024-05-20T22:30:46.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:46 smithi055 ceph-mon[29498]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:30:46.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:46 smithi055 ceph-mon[29498]: osdmap e48: 7 total, 7 up, 7 in 2024-05-20T22:30:47.367 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: osdmap e49: 7 total, 7 up, 7 in 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:47 smithi138 ceph-mon[25673]: osdmap e50: 7 total, 7 up, 7 in 2024-05-20T22:30:47.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: osdmap e49: 7 total, 7 up, 7 in 2024-05-20T22:30:47.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:47.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:47.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[24757]: osdmap e50: 7 total, 7 up, 7 in 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: osdmap e49: 7 total, 7 up, 7 in 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:47.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:47.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:30:47.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:47.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:47.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:47.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:30:47.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:47 smithi055 ceph-mon[29498]: osdmap e50: 7 total, 7 up, 7 in 2024-05-20T22:30:48.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:48 smithi138 ceph-mon[25673]: Detected new or changed devices on smithi138 2024-05-20T22:30:48.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:48 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi138 to 5696M 2024-05-20T22:30:48.368 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:48 smithi138 ceph-mon[25673]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:30:48.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:30:48.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:30:48.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:48 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:48.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:48 smithi138 ceph-mon[25673]: osdmap e51: 7 total, 7 up, 7 in 2024-05-20T22:30:48.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[24757]: Detected new or changed devices on smithi138 2024-05-20T22:30:48.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi138 to 5696M 2024-05-20T22:30:48.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[24757]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:30:48.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:30:48.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:30:48.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:48.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[24757]: osdmap e51: 7 total, 7 up, 7 in 2024-05-20T22:30:48.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[29498]: Detected new or changed devices on smithi138 2024-05-20T22:30:48.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi138 to 5696M 2024-05-20T22:30:48.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[29498]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:30:48.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:30:48.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:30:48.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:48.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:48 smithi055 ceph-mon[29498]: osdmap e51: 7 total, 7 up, 7 in 2024-05-20T22:30:49.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:49 smithi055 ceph-mon[24757]: from='client.34283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:49.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:49 smithi055 ceph-mon[29498]: from='client.34283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:49.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:49 smithi138 ceph-mon[25673]: from='client.34283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:50.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[24757]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:30:50.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/2046418394' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6708b4f8-c55d-4ef7-b2b2-3c17e616de52"}]: dispatch 2024-05-20T22:30:50.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6708b4f8-c55d-4ef7-b2b2-3c17e616de52"}]: dispatch 2024-05-20T22:30:50.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[24757]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6708b4f8-c55d-4ef7-b2b2-3c17e616de52"}]': finished 2024-05-20T22:30:50.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[24757]: osdmap e52: 8 total, 7 up, 8 in 2024-05-20T22:30:50.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:30:50.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[29498]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:30:50.640 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/2046418394' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6708b4f8-c55d-4ef7-b2b2-3c17e616de52"}]: dispatch 2024-05-20T22:30:50.640 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6708b4f8-c55d-4ef7-b2b2-3c17e616de52"}]: dispatch 2024-05-20T22:30:50.640 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[29498]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6708b4f8-c55d-4ef7-b2b2-3c17e616de52"}]': finished 2024-05-20T22:30:50.640 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[29498]: osdmap e52: 8 total, 7 up, 8 in 2024-05-20T22:30:50.640 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:50 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:30:50.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:50 smithi138 ceph-mon[25673]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:30:50.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:50 smithi138 ceph-mon[25673]: from='client.? 172.21.15.138:0/2046418394' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6708b4f8-c55d-4ef7-b2b2-3c17e616de52"}]: dispatch 2024-05-20T22:30:50.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:50 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6708b4f8-c55d-4ef7-b2b2-3c17e616de52"}]: dispatch 2024-05-20T22:30:50.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:50 smithi138 ceph-mon[25673]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6708b4f8-c55d-4ef7-b2b2-3c17e616de52"}]': finished 2024-05-20T22:30:50.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:50 smithi138 ceph-mon[25673]: osdmap e52: 8 total, 7 up, 8 in 2024-05-20T22:30:50.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:50 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:30:51.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:51 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/2046597268' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:30:51.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:51 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/2046597268' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:30:51.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:51 smithi138 ceph-mon[25673]: from='client.? 172.21.15.138:0/2046597268' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:30:52.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:52 smithi055 ceph-mon[24757]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-05-20T22:30:52.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:52 smithi055 ceph-mon[29498]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-05-20T22:30:52.740 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:52 smithi138 ceph-mon[25673]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-05-20T22:30:54.708 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:54 smithi138 ceph-mon[25673]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-20T22:30:54.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:54 smithi055 ceph-mon[24757]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-20T22:30:54.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:54 smithi055 ceph-mon[29498]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-20T22:30:55.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:55 smithi055 ceph-mon[24757]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-05-20T22:30:55.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:55 smithi055 ceph-mon[29498]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-05-20T22:30:55.797 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:55 smithi138 ceph-mon[25673]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-05-20T22:30:56.715 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:56 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:30:56.716 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:56 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:56.716 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:56 smithi138 ceph-mon[25673]: Deploying daemon osd.7 on smithi138 2024-05-20T22:30:56.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:56 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:30:56.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:56 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:56.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:56 smithi055 ceph-mon[24757]: Deploying daemon osd.7 on smithi138 2024-05-20T22:30:56.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:56 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:30:56.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:56 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:56.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:56 smithi055 ceph-mon[29498]: Deploying daemon osd.7 on smithi138 2024-05-20T22:30:58.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:30:58 smithi055 ceph-mon[24757]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-05-20T22:30:58.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:30:58 smithi055 ceph-mon[29498]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-05-20T22:30:58.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:30:58 smithi138 ceph-mon[25673]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-05-20T22:31:00.252 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:00 smithi138 ceph-mon[25673]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-20T22:31:00.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:00 smithi055 ceph-mon[24757]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-20T22:31:00.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:00 smithi055 ceph-mon[29498]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-20T22:31:01.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:01.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:01 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:01.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:31:01.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:31:01.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:01.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:01.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:01.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:01 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:02.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:02 smithi138 ceph-mon[25673]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:02.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:02 smithi055 ceph-mon[24757]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:02.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:02 smithi055 ceph-mon[29498]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:03.541 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 7 on host 'smithi138' 2024-05-20T22:31:04.165 DEBUG:teuthology.orchestra.run.smithi138:osd.7> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.7.service 2024-05-20T22:31:04.167 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-05-20T22:31:04.168 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd stat -f json 2024-05-20T22:31:04.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:04 smithi138 ceph-mon[25673]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:04.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:04 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:04.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:04 smithi138 ceph-mon[25673]: from='osd.7 [v2:172.21.15.138:6824/3609018365,v1:172.21.15.138:6825/3609018365]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:31:04.412 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:04 smithi138 ceph-mon[25673]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:31:04.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:04 smithi055 ceph-mon[24757]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:04.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:04 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:04.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:04 smithi055 ceph-mon[24757]: from='osd.7 [v2:172.21.15.138:6824/3609018365,v1:172.21.15.138:6825/3609018365]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:31:04.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:04 smithi055 ceph-mon[24757]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:31:04.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:04 smithi055 ceph-mon[29498]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:04.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:04 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:04.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:04 smithi055 ceph-mon[29498]: from='osd.7 [v2:172.21.15.138:6824/3609018365,v1:172.21.15.138:6825/3609018365]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:31:04.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:04 smithi055 ceph-mon[29498]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:31:05.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:05 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:05.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:05 smithi138 ceph-mon[25673]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T22:31:05.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:05 smithi138 ceph-mon[25673]: osdmap e53: 8 total, 7 up, 8 in 2024-05-20T22:31:05.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:05 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:05.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:05 smithi138 ceph-mon[25673]: from='osd.7 [v2:172.21.15.138:6824/3609018365,v1:172.21.15.138:6825/3609018365]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:31:05.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:05 smithi138 ceph-mon[25673]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:31:05.564 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 22:31:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[37732]: 2024-05-20T22:31:05.559+0000 7eff1df60700 -1 osd.7 0 waiting for initial osdmap 2024-05-20T22:31:05.564 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 22:31:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[37732]: 2024-05-20T22:31:05.566+0000 7eff190f8700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:31:05.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:05.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[24757]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T22:31:05.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[24757]: osdmap e53: 8 total, 7 up, 8 in 2024-05-20T22:31:05.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:05.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[24757]: from='osd.7 [v2:172.21.15.138:6824/3609018365,v1:172.21.15.138:6825/3609018365]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:31:05.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[24757]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:31:05.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:05.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[29498]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T22:31:05.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[29498]: osdmap e53: 8 total, 7 up, 8 in 2024-05-20T22:31:05.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:05.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[29498]: from='osd.7 [v2:172.21.15.138:6824/3609018365,v1:172.21.15.138:6825/3609018365]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:31:05.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:05 smithi055 ceph-mon[29498]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T22:31:06.465 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:31:06.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:06 smithi138 ceph-mon[25673]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:06 smithi138 ceph-mon[25673]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:31:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:06 smithi138 ceph-mon[25673]: osdmap e54: 8 total, 7 up, 8 in 2024-05-20T22:31:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:06 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:06.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:06 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:06.663 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[24757]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:06.663 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[24757]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:31:06.663 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[24757]: osdmap e54: 8 total, 7 up, 8 in 2024-05-20T22:31:06.663 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:06.663 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:06.663 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[29498]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:06.664 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[29498]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-05-20T22:31:06.664 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[29498]: osdmap e54: 8 total, 7 up, 8 in 2024-05-20T22:31:06.664 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:06.664 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:06 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:07.115 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1716244244,"num_in_osds":8,"osd_in_since":1716244249,"num_remapped_pgs":0} 2024-05-20T22:31:07.681 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:07 smithi138 ceph-mon[25673]: purged_snaps scrub starts 2024-05-20T22:31:07.681 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:07 smithi138 ceph-mon[25673]: purged_snaps scrub ok 2024-05-20T22:31:07.681 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:07 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2942851981' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T22:31:07.681 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:07 smithi138 ceph-mon[25673]: osd.7 [v2:172.21.15.138:6824/3609018365,v1:172.21.15.138:6825/3609018365] boot 2024-05-20T22:31:07.681 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:07 smithi138 ceph-mon[25673]: osdmap e55: 8 total, 8 up, 8 in 2024-05-20T22:31:07.681 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:07 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:07.682 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:07 smithi138 ceph-mon[25673]: osdmap e56: 8 total, 8 up, 8 in 2024-05-20T22:31:07.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[24757]: purged_snaps scrub starts 2024-05-20T22:31:07.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[24757]: purged_snaps scrub ok 2024-05-20T22:31:07.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2942851981' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T22:31:07.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[24757]: osd.7 [v2:172.21.15.138:6824/3609018365,v1:172.21.15.138:6825/3609018365] boot 2024-05-20T22:31:07.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[24757]: osdmap e55: 8 total, 8 up, 8 in 2024-05-20T22:31:07.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:07.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[24757]: osdmap e56: 8 total, 8 up, 8 in 2024-05-20T22:31:07.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[29498]: purged_snaps scrub starts 2024-05-20T22:31:07.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[29498]: purged_snaps scrub ok 2024-05-20T22:31:07.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2942851981' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T22:31:07.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[29498]: osd.7 [v2:172.21.15.138:6824/3609018365,v1:172.21.15.138:6825/3609018365] boot 2024-05-20T22:31:07.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[29498]: osdmap e55: 8 total, 8 up, 8 in 2024-05-20T22:31:07.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:07.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:07 smithi055 ceph-mon[29498]: osdmap e56: 8 total, 8 up, 8 in 2024-05-20T22:31:08.116 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd stat -f json 2024-05-20T22:31:08.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:08.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: Detected new or changed devices on smithi138 2024-05-20T22:31:08.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi138 to 4272M 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:08.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:08.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[29498]: osdmap e57: 8 total, 8 up, 8 in 2024-05-20T22:31:08.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:08.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: Detected new or changed devices on smithi138 2024-05-20T22:31:08.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi138 to 4272M 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:08 smithi055 ceph-mon[24757]: osdmap e57: 8 total, 8 up, 8 in 2024-05-20T22:31:08.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:31:08.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: Detected new or changed devices on smithi138 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi138 to 4272M 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:08.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:08 smithi138 ceph-mon[25673]: osdmap e57: 8 total, 8 up, 8 in 2024-05-20T22:31:10.210 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:31:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:10 smithi055 ceph-mon[24757]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:31:10.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:10 smithi055 ceph-mon[24757]: osdmap e58: 8 total, 8 up, 8 in 2024-05-20T22:31:10.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:10 smithi055 ceph-mon[29498]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:31:10.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:10 smithi055 ceph-mon[29498]: osdmap e58: 8 total, 8 up, 8 in 2024-05-20T22:31:10.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:10 smithi138 ceph-mon[25673]: pgmap v171: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:31:10.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:10 smithi138 ceph-mon[25673]: osdmap e58: 8 total, 8 up, 8 in 2024-05-20T22:31:10.807 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1716244266,"num_in_osds":8,"osd_in_since":1716244249,"num_remapped_pgs":0} 2024-05-20T22:31:10.807 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T22:31:11.032 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:11.469 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:11 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1261378518' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T22:31:11.469 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:11 smithi055 ceph-mon[24757]: osdmap e59: 8 total, 8 up, 8 in 2024-05-20T22:31:11.469 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:11 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1261378518' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T22:31:11.469 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:11 smithi055 ceph-mon[29498]: osdmap e59: 8 total, 8 up, 8 in 2024-05-20T22:31:11.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:11 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1261378518' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T22:31:11.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:11 smithi138 ceph-mon[25673]: osdmap e59: 8 total, 8 up, 8 in 2024-05-20T22:31:12.388 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:12 smithi055 ceph-mon[24757]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:31:12.388 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:12 smithi055 ceph-mon[29498]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:31:12.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:12 smithi138 ceph-mon[25673]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:31:12.792 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:31:12.792 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":59,"fsid":"dd0bace8-16f7-11ef-bc9a-c7b262605968","created":"2024-05-20T22:26:12.668590+0000","modified":"2024-05-20T22:31:10.175863+0000","last_up_change":"2024-05-20T22:31:06.553955+0000","last_in_change":"2024-05-20T22:30:49.964531+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T22:29:13.199964+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"32a19f07-edab-4360-a0df-a91d9239763e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6803","nonce":42457402}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6805","nonce":42457402}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6809","nonce":42457402}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6807","nonce":42457402}]},"public_addr":"172.21.15.55:6803/42457402","cluster_addr":"172.21.15.55:6805/42457402","heartbeat_back_addr":"172.21.15.55:6809/42457402","heartbeat_front_addr":"172.21.15.55:6807/42457402","state":["exists","up"]},{"osd":1,"uuid":"ca54976a-fed3-469f-8667-f4047f13689a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6811","nonce":3429244324}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6813","nonce":3429244324}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6817","nonce":3429244324}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6815","nonce":3429244324}]},"public_addr":"172.21.15.55:6811/3429244324","cluster_addr":"172.21.15.55:6813/3429244324","heartbeat_back_addr":"172.21.15.55:6817/3429244324","heartbeat_front_addr":"172.21.15.55:6815/3429244324","state":["exists","up"]},{"osd":2,"uuid":"a10157ad-79cb-4cf7-8422-f68f5f788e44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6819","nonce":3539256308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6821","nonce":3539256308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6825","nonce":3539256308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6823","nonce":3539256308}]},"public_addr":"172.21.15.55:6819/3539256308","cluster_addr":"172.21.15.55:6821/3539256308","heartbeat_back_addr":"172.21.15.55:6825/3539256308","heartbeat_front_addr":"172.21.15.55:6823/3539256308","state":["exists","up"]},{"osd":3,"uuid":"39cc9d6e-81f4-4722-a776-662682ced296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6827","nonce":1051646179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6829","nonce":1051646179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6833","nonce":1051646179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6831","nonce":1051646179}]},"public_addr":"172.21.15.55:6827/1051646179","cluster_addr":"172.21.15.55:6829/1051646179","heartbeat_back_addr":"172.21.15.55:6833/1051646179","heartbeat_front_addr":"172.21.15.55:6831/1051646179","state":["exists","up"]},{"osd":4,"uuid":"100258f9-7e57-450f-b734-60a4ef197a15","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6801","nonce":2210516721}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6803","nonce":2210516721}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6807","nonce":2210516721}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6805","nonce":2210516721}]},"public_addr":"172.21.15.138:6801/2210516721","cluster_addr":"172.21.15.138:6803/2210516721","heartbeat_back_addr":"172.21.15.138:6807/2210516721","heartbeat_front_addr":"172.21.15.138:6805/2210516721","state":["exists","up"]},{"osd":5,"uuid":"db0c9b42-af8d-4903-9368-17f97b5bf09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6809","nonce":2265959642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6811","nonce":2265959642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6815","nonce":2265959642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6813","nonce":2265959642}]},"public_addr":"172.21.15.138:6809/2265959642","cluster_addr":"172.21.15.138:6811/2265959642","heartbeat_back_addr":"172.21.15.138:6815/2265959642","heartbeat_front_addr":"172.21.15.138:6813/2265959642","state":["exists","up"]},{"osd":6,"uuid":"13c8d3de-41af-430a-912c-cdaab84f62d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6817","nonce":3675925959}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6819","nonce":3675925959}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6823","nonce":3675925959}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6821","nonce":3675925959}]},"public_addr":"172.21.15.138:6817/3675925959","cluster_addr":"172.21.15.138:6819/3675925959","heartbeat_back_addr":"172.21.15.138:6823/3675925959","heartbeat_front_addr":"172.21.15.138:6821/3675925959","state":["exists","up"]},{"osd":7,"uuid":"6708b4f8-c55d-4ef7-b2b2-3c17e616de52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6825","nonce":3609018365}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6827","nonce":3609018365}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6831","nonce":3609018365}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6829","nonce":3609018365}]},"public_addr":"172.21.15.138:6825/3609018365","cluster_addr":"172.21.15.138:6827/3609018365","heartbeat_back_addr":"172.21.15.138:6831/3609018365","heartbeat_front_addr":"172.21.15.138:6829/3609018365","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:28:20.541989+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:28:44.595578+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:29:09.365986+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:29:35.335222+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:29:57.122210+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:30:19.692314+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:30:42.599713+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:31:04.900756+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:6801/429623249":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/3886325575":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/1787159779":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/1640123401":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/3211423299":"2024-05-21T22:26:38.508043+0000","172.21.15.55:6800/3728861281":"2024-05-21T22:26:38.508043+0000","172.21.15.55:6800/429623249":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/1882487112":"2024-05-21T22:26:38.508043+0000","172.21.15.55:0/1950099066":"2024-05-21T22:26:38.508043+0000","172.21.15.55:6801/3728861281":"2024-05-21T22:26:38.508043+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T22:31:13.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:13 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3192360243' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:31:13.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:13 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3192360243' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:31:13.596 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-20T22:29:13.199964+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-05-20T22:31:13.596 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-05-20T22:31:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:13 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3192360243' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:31:13.821 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:14.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:14 smithi055 ceph-mon[24757]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-20T22:31:14.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:14 smithi055 ceph-mon[29498]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-20T22:31:14.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:14 smithi138 ceph-mon[25673]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-20T22:31:15.650 INFO:teuthology.orchestra.run.smithi055.stdout:pg_num: 1 2024-05-20T22:31:16.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:16 smithi055 ceph-mon[29498]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-20T22:31:16.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:16 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3349295213' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T22:31:16.387 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:16 smithi055 ceph-mon[24757]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-20T22:31:16.387 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:16 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3349295213' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T22:31:16.389 INFO:tasks.cephadm:Adding prometheus.a on smithi138 2024-05-20T22:31:16.389 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch apply prometheus '1;smithi138=a' 2024-05-20T22:31:16.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:16 smithi138 ceph-mon[25673]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-20T22:31:16.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:16 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3349295213' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T22:31:18.161 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled prometheus update... 2024-05-20T22:31:18.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:18 smithi138 ceph-mon[25673]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:18.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:18 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:18.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:18 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:18.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:18 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:18.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:18 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:18.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:18 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:18.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:18 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-20T22:31:18.710 DEBUG:teuthology.orchestra.run.smithi138:prometheus.a> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service 2024-05-20T22:31:18.712 INFO:tasks.cephadm:Adding node-exporter.a on smithi055 2024-05-20T22:31:18.712 INFO:tasks.cephadm:Adding node-exporter.b on smithi138 2024-05-20T22:31:18.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch apply node-exporter '2;smithi055=a;smithi138=b' 2024-05-20T22:31:18.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[29498]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[24757]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:18.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:18.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:18.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' 2024-05-20T22:31:18.738 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:18 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-20T22:31:19.288 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ignoring --setuser ceph since I am not root 2024-05-20T22:31:19.288 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ignoring --setgroup ceph since I am not root 2024-05-20T22:31:19.288 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:19 smithi138 ceph-mon[25673]: from='client.34325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:19.288 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:19 smithi138 ceph-mon[25673]: Saving service prometheus spec with placement smithi138=a;count:1 2024-05-20T22:31:19.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:19 smithi138 ceph-mon[25673]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-20T22:31:19.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:19 smithi138 ceph-mon[25673]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-20T22:31:19.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:19 smithi055 ceph-mon[24757]: from='client.34325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:19.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:19 smithi055 ceph-mon[24757]: Saving service prometheus spec with placement smithi138=a;count:1 2024-05-20T22:31:19.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:19 smithi055 ceph-mon[24757]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-20T22:31:19.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:19 smithi055 ceph-mon[24757]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-20T22:31:19.487 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ignoring --setuser ceph since I am not root 2024-05-20T22:31:19.487 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ignoring --setgroup ceph since I am not root 2024-05-20T22:31:19.487 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:19 smithi055 ceph-mgr[24993]: -- 172.21.15.55:0/1538087966 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55732750c340 con 0x5573274e4c00 2024-05-20T22:31:19.487 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:19.385+0000 7f5c18391000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:31:19.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:19 smithi055 ceph-mon[29498]: from='client.34325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:19.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:19 smithi055 ceph-mon[29498]: Saving service prometheus spec with placement smithi138=a;count:1 2024-05-20T22:31:19.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:19 smithi055 ceph-mon[29498]: from='mgr.14152 172.21.15.55:0/682908035' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-20T22:31:19.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:19 smithi055 ceph-mon[29498]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-20T22:31:19.591 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:19.369+0000 7fcf945e4000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:31:19.591 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:19.497+0000 7fcf945e4000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:31:19.592 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:19.596+0000 7fcf945e4000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:31:19.933 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:19.518+0000 7f5c18391000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:31:19.933 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:19.619+0000 7f5c18391000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:31:19.933 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:19.937+0000 7f5c18391000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:31:20.186 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:19.910+0000 7fcf945e4000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:31:20.186 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:19.993+0000 7fcf945e4000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:31:20.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:20 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:20.023+0000 7f5c18391000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:31:21.478 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:21.224+0000 7f5c18391000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:31:21.478 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:21.396+0000 7f5c18391000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:31:21.609 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:21.125+0000 7fcf945e4000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:31:21.609 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:21.288+0000 7fcf945e4000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:31:21.609 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:21.369+0000 7fcf945e4000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:31:21.736 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:21.482+0000 7f5c18391000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:31:22.097 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:21.681+0000 7fcf945e4000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:31:22.098 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:21.771+0000 7fcf945e4000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:31:22.098 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:21.864+0000 7fcf945e4000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:31:22.098 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:22.102+0000 7fcf945e4000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:31:22.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:21.810+0000 7f5c18391000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:31:22.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:21.904+0000 7f5c18391000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:31:22.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:22.001+0000 7f5c18391000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:31:22.708 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:22.253+0000 7f5c18391000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:31:22.859 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:22.538+0000 7fcf945e4000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:31:22.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:22.713+0000 7f5c18391000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:31:23.359 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:23.022+0000 7fcf945e4000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:31:23.359 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:23.115+0000 7fcf945e4000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:31:23.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:23.218+0000 7f5c18391000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:31:23.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:23.316+0000 7f5c18391000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:31:23.859 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:23.406+0000 7fcf945e4000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:31:23.859 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:23.495+0000 7fcf945e4000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:31:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:23.624+0000 7f5c18391000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:31:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:23.717+0000 7f5c18391000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:31:24.227 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:23.897+0000 7fcf945e4000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:31:24.228 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:24.028+0000 7fcf945e4000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:31:24.228 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:24.119+0000 7fcf945e4000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:31:24.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:24.137+0000 7f5c18391000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:31:24.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:24.276+0000 7f5c18391000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:31:24.487 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:24.373+0000 7f5c18391000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:31:24.609 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:24.232+0000 7fcf945e4000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:31:24.609 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:24.322+0000 7fcf945e4000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:31:24.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:24.494+0000 7f5c18391000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:31:24.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:24.586+0000 7f5c18391000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:31:25.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:24.863+0000 7fcf945e4000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:31:25.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:25 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:25.161+0000 7f5c18391000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:31:25.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[24757]: Standby manager daemon x restarted 2024-05-20T22:31:25.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[24757]: Standby manager daemon x started 2024-05-20T22:31:25.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:31:25.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:31:25.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:31:25.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:31:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[29498]: Standby manager daemon x restarted 2024-05-20T22:31:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[29498]: Standby manager daemon x started 2024-05-20T22:31:25.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:31:25.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:31:25.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:31:25.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:25 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:31:25.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:25 smithi138 ceph-mon[25673]: Standby manager daemon x restarted 2024-05-20T22:31:25.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:25 smithi138 ceph-mon[25673]: Standby manager daemon x started 2024-05-20T22:31:25.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:25 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:31:25.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:25 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:31:25.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:25 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:31:25.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:25 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1013019819' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:31:25.610 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:25.224+0000 7fcf945e4000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:31:25.610 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: [20/May/2024:22:31:25] ENGINE Bus STARTING 2024-05-20T22:31:25.610 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: CherryPy Checker: 2024-05-20T22:31:25.610 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: The Application mounted at '' has an empty config. 2024-05-20T22:31:25.610 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: 2024-05-20T22:31:25.610 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: [20/May/2024:22:31:25] ENGINE Serving on http://:::9283 2024-05-20T22:31:25.610 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: [20/May/2024:22:31:25] ENGINE Bus STARTED 2024-05-20T22:31:25.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:25 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:25.537+0000 7f5c18391000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:31:26.555 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:26 smithi055 ceph-mon[24757]: mgrmap e17: y(active, since 4m), standbys: x 2024-05-20T22:31:26.555 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:26 smithi055 ceph-mon[24757]: Active manager daemon y restarted 2024-05-20T22:31:26.555 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:26 smithi055 ceph-mon[24757]: Activating manager daemon y 2024-05-20T22:31:26.556 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:26 smithi055 ceph-mon[24757]: osdmap e60: 8 total, 8 up, 8 in 2024-05-20T22:31:26.556 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:26 smithi055 ceph-mon[29498]: mgrmap e17: y(active, since 4m), standbys: x 2024-05-20T22:31:26.556 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:26 smithi055 ceph-mon[29498]: Active manager daemon y restarted 2024-05-20T22:31:26.556 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:26 smithi055 ceph-mon[29498]: Activating manager daemon y 2024-05-20T22:31:26.556 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:26 smithi055 ceph-mon[29498]: osdmap e60: 8 total, 8 up, 8 in 2024-05-20T22:31:26.556 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:31:26] ENGINE Bus STARTING 2024-05-20T22:31:26.556 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: CherryPy Checker: 2024-05-20T22:31:26.556 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: The Application mounted at '' has an empty config. 2024-05-20T22:31:26.557 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:31:26.557 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:31:26] ENGINE Serving on http://:::9283 2024-05-20T22:31:26.557 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:31:26] ENGINE Bus STARTED 2024-05-20T22:31:26.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:26 smithi138 ceph-mon[25673]: mgrmap e17: y(active, since 4m), standbys: x 2024-05-20T22:31:26.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:26 smithi138 ceph-mon[25673]: Active manager daemon y restarted 2024-05-20T22:31:26.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:26 smithi138 ceph-mon[25673]: Activating manager daemon y 2024-05-20T22:31:26.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:26 smithi138 ceph-mon[25673]: osdmap e60: 8 total, 8 up, 8 in 2024-05-20T22:31:26.865 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:31:26] ENGINE Bus STARTING 2024-05-20T22:31:27.130 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:31:26] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:31:27.130 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:31:26] ENGINE Bus STARTED 2024-05-20T22:31:27.334 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled node-exporter update... 2024-05-20T22:31:27.405 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: mgrmap e18: y(active, starting, since 0.78263s), standbys: x 2024-05-20T22:31:27.405 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:31:27.405 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:27.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: Manager daemon y is now available 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:31:27.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: mgrmap e18: y(active, starting, since 0.78263s), standbys: x 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:31:27.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: Manager daemon y is now available 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:31:27.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: [20/May/2024:22:31:26] ENGINE Bus STARTING 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: [20/May/2024:22:31:26] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: [20/May/2024:22:31:26] ENGINE Bus STARTED 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: [20/May/2024:22:31:26] ENGINE Bus STARTING 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: [20/May/2024:22:31:26] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: [20/May/2024:22:31:26] ENGINE Bus STARTED 2024-05-20T22:31:27.410 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:27.527 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: mgrmap e18: y(active, starting, since 0.78263s), standbys: x 2024-05-20T22:31:27.527 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:31:27.527 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:31:27.527 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:31:27.527 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:31:27.527 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:31:27.527 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:31:27.527 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:31:27.527 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: Manager daemon y is now available 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:27.528 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:27.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:27.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:31:27.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:31:27.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:31:27.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:31:27.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: [20/May/2024:22:31:26] ENGINE Bus STARTING 2024-05-20T22:31:27.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: [20/May/2024:22:31:26] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:31:27.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: [20/May/2024:22:31:26] ENGINE Bus STARTED 2024-05-20T22:31:27.529 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:28.079 DEBUG:teuthology.orchestra.run.smithi055:node-exporter.a> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service 2024-05-20T22:31:28.081 DEBUG:teuthology.orchestra.run.smithi138:node-exporter.b> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service 2024-05-20T22:31:28.083 INFO:tasks.cephadm:Adding alertmanager.a on smithi055 2024-05-20T22:31:28.084 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch apply alertmanager '1;smithi055=a' 2024-05-20T22:31:28.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[29498]: mgrmap e19: y(active, since 1.78702s), standbys: x 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[29498]: from='client.34343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi055=a;smithi138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[29498]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[29498]: Saving service node-exporter spec with placement smithi055=a;smithi138=b;count:2 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[24757]: mgrmap e19: y(active, since 1.78702s), standbys: x 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[24757]: from='client.34343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi055=a;smithi138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[24757]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[24757]: Saving service node-exporter spec with placement smithi055=a;smithi138=b;count:2 2024-05-20T22:31:28.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:28.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:28 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:28 smithi138 ceph-mon[25673]: mgrmap e19: y(active, since 1.78702s), standbys: x 2024-05-20T22:31:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:28 smithi138 ceph-mon[25673]: from='client.34343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi055=a;smithi138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:28 smithi138 ceph-mon[25673]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:28 smithi138 ceph-mon[25673]: Saving service node-exporter spec with placement smithi055=a;smithi138=b;count:2 2024-05-20T22:31:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:28 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:28 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:28 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.479 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: mgrmap e20: y(active, since 2s), standbys: x 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: mgrmap e20: y(active, since 2s), standbys: x 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: mgrmap e20: y(active, since 2s), standbys: x 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:31:29.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:29.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:30.159 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled alertmanager update... 2024-05-20T22:31:31.020 DEBUG:teuthology.orchestra.run.smithi055:alertmanager.a> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@alertmanager.a.service 2024-05-20T22:31:31.023 INFO:tasks.cephadm:Adding grafana.a on smithi138 2024-05-20T22:31:31.023 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph orch apply grafana '1;smithi138=a' 2024-05-20T22:31:31.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:31 smithi055 ceph-mon[24757]: Deploying daemon prometheus.a on smithi138 2024-05-20T22:31:31.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:31 smithi055 ceph-mon[24757]: from='client.34373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi055=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:31.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:31 smithi055 ceph-mon[24757]: Saving service alertmanager spec with placement smithi055=a;count:1 2024-05-20T22:31:31.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:31 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:31.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:31 smithi055 ceph-mon[29498]: Deploying daemon prometheus.a on smithi138 2024-05-20T22:31:31.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:31 smithi055 ceph-mon[29498]: from='client.34373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi055=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:31.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:31 smithi055 ceph-mon[29498]: Saving service alertmanager spec with placement smithi055=a;count:1 2024-05-20T22:31:31.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:31 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:31.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:31 smithi138 ceph-mon[25673]: Deploying daemon prometheus.a on smithi138 2024-05-20T22:31:31.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:31 smithi138 ceph-mon[25673]: from='client.34373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi055=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:31.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:31 smithi138 ceph-mon[25673]: Saving service alertmanager spec with placement smithi055=a;count:1 2024-05-20T22:31:31.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:31 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:32.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:32 smithi055 ceph-mon[24757]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:32.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:32 smithi055 ceph-mon[29498]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:32.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:32 smithi138 ceph-mon[25673]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:33.814 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled grafana update... 2024-05-20T22:31:34.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:34 smithi055 ceph-mon[24757]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:34.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:34 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:34.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:34 smithi055 ceph-mon[29498]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:34.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:34 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:34.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:34 smithi138 ceph-mon[25673]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:34.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:34 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:34.772 DEBUG:teuthology.orchestra.run.smithi138:grafana.a> sudo journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@grafana.a.service 2024-05-20T22:31:34.774 INFO:tasks.cephadm:Setting up client nodes... 2024-05-20T22:31:34.774 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-20T22:31:35.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:35 smithi055 ceph-mon[24757]: from='client.34379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:35.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:35 smithi055 ceph-mon[24757]: Saving service grafana spec with placement smithi138=a;count:1 2024-05-20T22:31:35.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:35 smithi055 ceph-mon[29498]: from='client.34379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:35.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:35 smithi055 ceph-mon[29498]: Saving service grafana spec with placement smithi138=a;count:1 2024-05-20T22:31:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:35 smithi138 ceph-mon[25673]: from='client.34379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:35 smithi138 ceph-mon[25673]: Saving service grafana spec with placement smithi138=a;count:1 2024-05-20T22:31:36.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:36 smithi055 ceph-mon[24757]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:36.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:36 smithi055 ceph-mon[29498]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:36.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:36 smithi138 ceph-mon[25673]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:36.991 INFO:teuthology.orchestra.run.smithi055.stdout:[client.0] 2024-05-20T22:31:36.991 INFO:teuthology.orchestra.run.smithi055.stdout: key = AQBIz0tmIdH5OhAAhRBUXbGxyCpPVWkfZRjo5A== 2024-05-20T22:31:37.267 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:37 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2063223616' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:37.267 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:37 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:37.267 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:37 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T22:31:37.267 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:37 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2063223616' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:37.267 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:37 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:37.267 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:37 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T22:31:37.599 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-05-20T22:31:37.599 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-05-20T22:31:37.599 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-05-20T22:31:37.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:37 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2063223616' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:37.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:37 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:37.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:37 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T22:31:37.638 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-20T22:31:38.232 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:38 smithi138 ceph-mon[25673]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:38.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:38 smithi055 ceph-mon[24757]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:38.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:38 smithi055 ceph-mon[29498]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:40.460 INFO:teuthology.orchestra.run.smithi138.stdout:[client.1] 2024-05-20T22:31:40.460 INFO:teuthology.orchestra.run.smithi138.stdout: key = AQBMz0tmkeF8GxAAxTJivtb8LRh2tozPE8F/vA== 2024-05-20T22:31:40.461 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:40 smithi138 ceph-mon[25673]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:40.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:40 smithi055 ceph-mon[24757]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:40.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:40 smithi055 ceph-mon[29498]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:41.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:41 smithi055 ceph-mon[24757]: from='client.? 172.21.15.138:0/36042663' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:41.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:41 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:41.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:41 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T22:31:41.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:41 smithi055 ceph-mon[29498]: from='client.? 172.21.15.138:0/36042663' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:41.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:41 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:41.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:41 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T22:31:41.514 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:41 smithi138 ceph-mon[25673]: from='client.? 172.21.15.138:0/36042663' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:41.514 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:41 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:31:41.514 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:41 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T22:31:42.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:42 smithi138 ceph-mon[25673]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:42.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:42 smithi055 ceph-mon[24757]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:42.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:42 smithi055 ceph-mon[29498]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:42.884 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-05-20T22:31:42.884 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-05-20T22:31:42.884 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-05-20T22:31:42.924 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-05-20T22:31:42.924 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-05-20T22:31:42.924 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph mgr dump --format=json 2024-05-20T22:31:43.157 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:43.333 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 podman[40706]: 2024-05-20 22:31:43.314173203 +0000 UTC m=+0.364357375 container start 18fd7fbd48b74b1f552c24052cb4a5abd77e1d8287f43eda609422de968f3b15 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:31:43.609 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.338Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.338Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.338Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.338Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi138 (none))" 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.338Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.338Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.340Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.341Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.341Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.343Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-20T22:31:43.610 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.343Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.44µs 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.343Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.343Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.343Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.545µs wal_replay_duration=166.699µs total_replay_duration=213.09µs 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.343Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.344Z caller=main.go:947 level=info msg="TSDB started" 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.344Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.359Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.067727ms db_storage=708ns remote_storage=1.582µs web_handler=250ns query_engine=552ns scrape=412.915µs scrape_sd=21.783µs notify=810ns notify_sd=1.61µs rules=14.27234ms 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:31:43.359Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 bash[40706]: 18fd7fbd48b74b1f552c24052cb4a5abd77e1d8287f43eda609422de968f3b15 2024-05-20T22:31:43.611 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:31:43 smithi138 systemd[1]: Started Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:31:44.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:44 smithi055 ceph-mon[29498]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:44.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:44 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:44.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:44 smithi055 ceph-mon[24757]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:44.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:44 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:44.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:44 smithi138 ceph-mon[25673]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:44.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:44 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:45.332 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:31:45.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:45 smithi055 ceph-mon[24757]: Deploying daemon node-exporter.a on smithi055 2024-05-20T22:31:45.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:45 smithi055 ceph-mon[29498]: Deploying daemon node-exporter.a on smithi055 2024-05-20T22:31:45.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:45 smithi138 ceph-mon[25673]: Deploying daemon node-exporter.a on smithi055 2024-05-20T22:31:45.748 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:45 smithi055 bash[44301]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-05-20T22:31:46.293 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":20,"active_gid":14433,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":159719434},{"type":"v1","addr":"172.21.15.55:6801","nonce":159719434}]},"active_addr":"172.21.15.55:6801/159719434","active_change":"2024-05-20T22:31:25.539592+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":34331,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":2831507036}]},{"addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":53762980}]},{"addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":2503496786}]},{"addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":1780260259}]}]}} 2024-05-20T22:31:46.299 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-05-20T22:31:46.299 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-05-20T22:31:46.299 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T22:31:46.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:46 smithi055 ceph-mon[24757]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:46.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:46 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/750610216' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:31:46.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:46 smithi055 ceph-mon[29498]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:46.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:46 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/750610216' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:31:46.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:46 smithi138 ceph-mon[25673]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:46.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:46 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/750610216' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:31:47.076 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:47.448 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:47 smithi055 ceph-mon[24757]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:47.448 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:47 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:47.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:47 smithi055 ceph-mon[29498]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:47.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:47 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:47.449 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:47 smithi055 bash[44301]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-05-20T22:31:47.736 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:47 smithi055 bash[44301]: Writing manifest to image destination 2024-05-20T22:31:47.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:47 smithi138 ceph-mon[25673]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:47.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:47 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:48.486 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:48 smithi055 podman[44301]: 2024-05-20 22:31:48.027149211 +0000 UTC m=+2.879565490 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-20T22:31:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:31:49.300 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":60,"fsid":"dd0bace8-16f7-11ef-bc9a-c7b262605968","created":"2024-05-20T22:26:12.668590+0000","modified":"2024-05-20T22:31:25.539177+0000","last_up_change":"2024-05-20T22:31:06.553955+0000","last_in_change":"2024-05-20T22:30:49.964531+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T22:29:13.199964+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"32a19f07-edab-4360-a0df-a91d9239763e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6803","nonce":42457402}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6805","nonce":42457402}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6809","nonce":42457402}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6807","nonce":42457402}]},"public_addr":"172.21.15.55:6803/42457402","cluster_addr":"172.21.15.55:6805/42457402","heartbeat_back_addr":"172.21.15.55:6809/42457402","heartbeat_front_addr":"172.21.15.55:6807/42457402","state":["exists","up"]},{"osd":1,"uuid":"ca54976a-fed3-469f-8667-f4047f13689a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6811","nonce":3429244324}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6813","nonce":3429244324}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6817","nonce":3429244324}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6815","nonce":3429244324}]},"public_addr":"172.21.15.55:6811/3429244324","cluster_addr":"172.21.15.55:6813/3429244324","heartbeat_back_addr":"172.21.15.55:6817/3429244324","heartbeat_front_addr":"172.21.15.55:6815/3429244324","state":["exists","up"]},{"osd":2,"uuid":"a10157ad-79cb-4cf7-8422-f68f5f788e44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6819","nonce":3539256308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6821","nonce":3539256308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6825","nonce":3539256308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6823","nonce":3539256308}]},"public_addr":"172.21.15.55:6819/3539256308","cluster_addr":"172.21.15.55:6821/3539256308","heartbeat_back_addr":"172.21.15.55:6825/3539256308","heartbeat_front_addr":"172.21.15.55:6823/3539256308","state":["exists","up"]},{"osd":3,"uuid":"39cc9d6e-81f4-4722-a776-662682ced296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6827","nonce":1051646179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6829","nonce":1051646179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6833","nonce":1051646179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6831","nonce":1051646179}]},"public_addr":"172.21.15.55:6827/1051646179","cluster_addr":"172.21.15.55:6829/1051646179","heartbeat_back_addr":"172.21.15.55:6833/1051646179","heartbeat_front_addr":"172.21.15.55:6831/1051646179","state":["exists","up"]},{"osd":4,"uuid":"100258f9-7e57-450f-b734-60a4ef197a15","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6801","nonce":2210516721}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6803","nonce":2210516721}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6807","nonce":2210516721}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6805","nonce":2210516721}]},"public_addr":"172.21.15.138:6801/2210516721","cluster_addr":"172.21.15.138:6803/2210516721","heartbeat_back_addr":"172.21.15.138:6807/2210516721","heartbeat_front_addr":"172.21.15.138:6805/2210516721","state":["exists","up"]},{"osd":5,"uuid":"db0c9b42-af8d-4903-9368-17f97b5bf09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6809","nonce":2265959642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6811","nonce":2265959642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6815","nonce":2265959642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6813","nonce":2265959642}]},"public_addr":"172.21.15.138:6809/2265959642","cluster_addr":"172.21.15.138:6811/2265959642","heartbeat_back_addr":"172.21.15.138:6815/2265959642","heartbeat_front_addr":"172.21.15.138:6813/2265959642","state":["exists","up"]},{"osd":6,"uuid":"13c8d3de-41af-430a-912c-cdaab84f62d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6817","nonce":3675925959}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6819","nonce":3675925959}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6823","nonce":3675925959}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6821","nonce":3675925959}]},"public_addr":"172.21.15.138:6817/3675925959","cluster_addr":"172.21.15.138:6819/3675925959","heartbeat_back_addr":"172.21.15.138:6823/3675925959","heartbeat_front_addr":"172.21.15.138:6821/3675925959","state":["exists","up"]},{"osd":7,"uuid":"6708b4f8-c55d-4ef7-b2b2-3c17e616de52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6825","nonce":3609018365}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6827","nonce":3609018365}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6831","nonce":3609018365}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6829","nonce":3609018365}]},"public_addr":"172.21.15.138:6825/3609018365","cluster_addr":"172.21.15.138:6827/3609018365","heartbeat_back_addr":"172.21.15.138:6831/3609018365","heartbeat_front_addr":"172.21.15.138:6829/3609018365","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:28:20.541989+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:28:44.595578+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:29:09.365986+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:29:35.335222+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:29:57.122210+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:30:19.692314+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:30:42.599713+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:31:04.900756+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/2680501974":"2024-05-21T22:31:25.539122+0000","172.21.15.55:0/2905501019":"2024-05-21T22:31:25.539122+0000","172.21.15.55:6801/429623249":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/3886325575":"2024-05-21T22:27:00.646462+0000","172.21.15.55:6801/388134085":"2024-05-21T22:31:25.539122+0000","172.21.15.55:0/1787159779":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/1640123401":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/3211423299":"2024-05-21T22:26:38.508043+0000","172.21.15.55:6800/388134085":"2024-05-21T22:31:25.539122+0000","172.21.15.55:0/2635876549":"2024-05-21T22:31:25.539122+0000","172.21.15.55:6800/3728861281":"2024-05-21T22:26:38.508043+0000","172.21.15.55:0/3366104370":"2024-05-21T22:31:25.539122+0000","172.21.15.55:6800/429623249":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/1882487112":"2024-05-21T22:26:38.508043+0000","172.21.15.55:0/1950099066":"2024-05-21T22:26:38.508043+0000","172.21.15.55:6801/3728861281":"2024-05-21T22:26:38.508043+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T22:31:49.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:49 smithi055 ceph-mon[24757]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:49.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:49 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3351706362' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:31:49.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:49 smithi055 ceph-mon[29498]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:49.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:49 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3351706362' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:31:49.666 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:49 smithi055 podman[44301]: 2024-05-20 22:31:49.282099153 +0000 UTC m=+4.134515431 container create 08e55b1e4140078e1bb52d97774f3d3a46ee2c0d80bb253bc28694b958674c63 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:31:49.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:49 smithi138 ceph-mon[25673]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:49.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:49 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3351706362' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:31:50.080 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-05-20T22:31:50.080 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T22:31:50.335 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 podman[44301]: 2024-05-20 22:31:50.23377818 +0000 UTC m=+5.086194466 container init 08e55b1e4140078e1bb52d97774f3d3a46ee2c0d80bb253bc28694b958674c63 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:31:50.335 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 podman[44301]: 2024-05-20 22:31:50.237352375 +0000 UTC m=+5.089768654 container start 08e55b1e4140078e1bb52d97774f3d3a46ee2c0d80bb253bc28694b958674c63 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:31:50.335 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.241Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-20T22:31:50.335 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.241Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-20T22:31:50.335 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=arp 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-20T22:31:50.336 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=edac 2024-05-20T22:31:50.337 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-20T22:31:50.337 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-20T22:31:50.337 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-20T22:31:50.337 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-20T22:31:50.337 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-20T22:31:50.337 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-20T22:31:50.337 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-20T22:31:50.337 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-20T22:31:50.337 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=os 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-20T22:31:50.338 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=stat 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=time 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=uname 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.242Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.243Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-20T22:31:50.339 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[44507]: ts=2024-05-20T22:31:50.243Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-20T22:31:50.340 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 bash[44301]: 08e55b1e4140078e1bb52d97774f3d3a46ee2c0d80bb253bc28694b958674c63 2024-05-20T22:31:50.394 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:50.646 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:31:50 smithi055 systemd[1]: Started Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:31:51.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:51 smithi138 bash[40938]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-05-20T22:31:51.687 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:51 smithi138 ceph-mon[25673]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:51.687 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:51 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:51.687 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:51 smithi138 ceph-mon[25673]: Deploying daemon node-exporter.b on smithi138 2024-05-20T22:31:51.702 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:51 smithi055 ceph-mon[24757]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:51.703 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:51 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:51.703 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:51 smithi055 ceph-mon[24757]: Deploying daemon node-exporter.b on smithi138 2024-05-20T22:31:51.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:51 smithi055 ceph-mon[29498]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:51.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:51 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:51.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:51 smithi055 ceph-mon[29498]: Deploying daemon node-exporter.b on smithi138 2024-05-20T22:31:52.109 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:51 smithi138 bash[40938]: Getting image source signatures 2024-05-20T22:31:52.109 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:51 smithi138 bash[40938]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-05-20T22:31:52.109 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:51 smithi138 bash[40938]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-05-20T22:31:52.109 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:51 smithi138 bash[40938]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-05-20T22:31:52.412 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:31:52.412 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":60,"fsid":"dd0bace8-16f7-11ef-bc9a-c7b262605968","created":"2024-05-20T22:26:12.668590+0000","modified":"2024-05-20T22:31:25.539177+0000","last_up_change":"2024-05-20T22:31:06.553955+0000","last_in_change":"2024-05-20T22:30:49.964531+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T22:29:13.199964+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"32a19f07-edab-4360-a0df-a91d9239763e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6803","nonce":42457402}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6805","nonce":42457402}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6809","nonce":42457402}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":42457402},{"type":"v1","addr":"172.21.15.55:6807","nonce":42457402}]},"public_addr":"172.21.15.55:6803/42457402","cluster_addr":"172.21.15.55:6805/42457402","heartbeat_back_addr":"172.21.15.55:6809/42457402","heartbeat_front_addr":"172.21.15.55:6807/42457402","state":["exists","up"]},{"osd":1,"uuid":"ca54976a-fed3-469f-8667-f4047f13689a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6811","nonce":3429244324}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6813","nonce":3429244324}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6817","nonce":3429244324}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":3429244324},{"type":"v1","addr":"172.21.15.55:6815","nonce":3429244324}]},"public_addr":"172.21.15.55:6811/3429244324","cluster_addr":"172.21.15.55:6813/3429244324","heartbeat_back_addr":"172.21.15.55:6817/3429244324","heartbeat_front_addr":"172.21.15.55:6815/3429244324","state":["exists","up"]},{"osd":2,"uuid":"a10157ad-79cb-4cf7-8422-f68f5f788e44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6819","nonce":3539256308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6821","nonce":3539256308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6825","nonce":3539256308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":3539256308},{"type":"v1","addr":"172.21.15.55:6823","nonce":3539256308}]},"public_addr":"172.21.15.55:6819/3539256308","cluster_addr":"172.21.15.55:6821/3539256308","heartbeat_back_addr":"172.21.15.55:6825/3539256308","heartbeat_front_addr":"172.21.15.55:6823/3539256308","state":["exists","up"]},{"osd":3,"uuid":"39cc9d6e-81f4-4722-a776-662682ced296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6827","nonce":1051646179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6829","nonce":1051646179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6833","nonce":1051646179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1051646179},{"type":"v1","addr":"172.21.15.55:6831","nonce":1051646179}]},"public_addr":"172.21.15.55:6827/1051646179","cluster_addr":"172.21.15.55:6829/1051646179","heartbeat_back_addr":"172.21.15.55:6833/1051646179","heartbeat_front_addr":"172.21.15.55:6831/1051646179","state":["exists","up"]},{"osd":4,"uuid":"100258f9-7e57-450f-b734-60a4ef197a15","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6801","nonce":2210516721}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6803","nonce":2210516721}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6807","nonce":2210516721}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":2210516721},{"type":"v1","addr":"172.21.15.138:6805","nonce":2210516721}]},"public_addr":"172.21.15.138:6801/2210516721","cluster_addr":"172.21.15.138:6803/2210516721","heartbeat_back_addr":"172.21.15.138:6807/2210516721","heartbeat_front_addr":"172.21.15.138:6805/2210516721","state":["exists","up"]},{"osd":5,"uuid":"db0c9b42-af8d-4903-9368-17f97b5bf09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6809","nonce":2265959642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6811","nonce":2265959642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6815","nonce":2265959642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":2265959642},{"type":"v1","addr":"172.21.15.138:6813","nonce":2265959642}]},"public_addr":"172.21.15.138:6809/2265959642","cluster_addr":"172.21.15.138:6811/2265959642","heartbeat_back_addr":"172.21.15.138:6815/2265959642","heartbeat_front_addr":"172.21.15.138:6813/2265959642","state":["exists","up"]},{"osd":6,"uuid":"13c8d3de-41af-430a-912c-cdaab84f62d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6817","nonce":3675925959}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6819","nonce":3675925959}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6823","nonce":3675925959}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3675925959},{"type":"v1","addr":"172.21.15.138:6821","nonce":3675925959}]},"public_addr":"172.21.15.138:6817/3675925959","cluster_addr":"172.21.15.138:6819/3675925959","heartbeat_back_addr":"172.21.15.138:6823/3675925959","heartbeat_front_addr":"172.21.15.138:6821/3675925959","state":["exists","up"]},{"osd":7,"uuid":"6708b4f8-c55d-4ef7-b2b2-3c17e616de52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6825","nonce":3609018365}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6827","nonce":3609018365}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6831","nonce":3609018365}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":3609018365},{"type":"v1","addr":"172.21.15.138:6829","nonce":3609018365}]},"public_addr":"172.21.15.138:6825/3609018365","cluster_addr":"172.21.15.138:6827/3609018365","heartbeat_back_addr":"172.21.15.138:6831/3609018365","heartbeat_front_addr":"172.21.15.138:6829/3609018365","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:28:20.541989+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:28:44.595578+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:29:09.365986+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:29:35.335222+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:29:57.122210+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:30:19.692314+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:30:42.599713+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:31:04.900756+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/2680501974":"2024-05-21T22:31:25.539122+0000","172.21.15.55:0/2905501019":"2024-05-21T22:31:25.539122+0000","172.21.15.55:6801/429623249":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/3886325575":"2024-05-21T22:27:00.646462+0000","172.21.15.55:6801/388134085":"2024-05-21T22:31:25.539122+0000","172.21.15.55:0/1787159779":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/1640123401":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/3211423299":"2024-05-21T22:26:38.508043+0000","172.21.15.55:6800/388134085":"2024-05-21T22:31:25.539122+0000","172.21.15.55:0/2635876549":"2024-05-21T22:31:25.539122+0000","172.21.15.55:6800/3728861281":"2024-05-21T22:26:38.508043+0000","172.21.15.55:0/3366104370":"2024-05-21T22:31:25.539122+0000","172.21.15.55:6800/429623249":"2024-05-21T22:27:00.646462+0000","172.21.15.55:0/1882487112":"2024-05-21T22:26:38.508043+0000","172.21.15.55:0/1950099066":"2024-05-21T22:26:38.508043+0000","172.21.15.55:6801/3728861281":"2024-05-21T22:26:38.508043+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T22:31:53.051 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-05-20T22:31:53.051 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-05-20T22:31:53.052 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-05-20T22:31:53.052 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-05-20T22:31:53.052 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-05-20T22:31:53.053 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-05-20T22:31:53.053 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-05-20T22:31:53.053 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-05-20T22:31:53.315 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:53 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1722994002' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:31:53.315 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:53 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1722994002' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:31:53.486 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:53.486 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:53.487 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:53.487 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:53.499 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:53.524 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:53.540 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:53.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1722994002' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:31:53.583 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 bash[40938]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-05-20T22:31:53.583 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 bash[40938]: Writing manifest to image destination 2024-05-20T22:31:53.860 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 podman[40938]: 2024-05-20 22:31:53.548978734 +0000 UTC m=+2.386437768 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-20T22:31:53.860 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 podman[40938]: 2024-05-20 22:31:53.614109508 +0000 UTC m=+2.451568518 container create 8a4a20d56038e441c8726617c0418ff5283e9dcae8b99772c4ac6473d5b24b10 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:31:53.860 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 podman[40938]: 2024-05-20 22:31:53.791147231 +0000 UTC m=+2.628606240 container init 8a4a20d56038e441c8726617c0418ff5283e9dcae8b99772c4ac6473d5b24b10 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:31:53.860 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 podman[40938]: 2024-05-20 22:31:53.793951848 +0000 UTC m=+2.631410876 container start 8a4a20d56038e441c8726617c0418ff5283e9dcae8b99772c4ac6473d5b24b10 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.798Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.798Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.798Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.798Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=arp 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-20T22:31:53.861 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-20T22:31:53.862 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-20T22:31:53.862 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-20T22:31:53.862 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=edac 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-20T22:31:53.863 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=os 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-20T22:31:53.864 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=stat 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=time 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=uname 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-20T22:31:53.865 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[40987]: ts=2024-05-20T22:31:53.799Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-20T22:31:54.185 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 bash[40938]: 8a4a20d56038e441c8726617c0418ff5283e9dcae8b99772c4ac6473d5b24b10 2024-05-20T22:31:54.185 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:31:53 smithi138 systemd[1]: Started Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:31:54.483 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:54 smithi138 ceph-mon[25673]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:54.483 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:54 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:54.483 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:54 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:54.483 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:54 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:54.483 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:54 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[24757]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:54.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[29498]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:54.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:54.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:54.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:54.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:54 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:55.329 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:31:56.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:31:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:31:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:31:56.436 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:56 smithi138 ceph-mon[25673]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:56.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:56 smithi055 ceph-mon[24757]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:56.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:56 smithi055 ceph-mon[29498]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:57.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:31:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:31:57] "GET /metrics HTTP/1.1" 200 191491 "" "Prometheus/2.33.4" 2024-05-20T22:31:57.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:57 smithi055 ceph-mon[24757]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:57.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:57 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:57.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:57 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:57 smithi055 ceph-mon[29498]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:57 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:57 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:57.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:57 smithi138 ceph-mon[25673]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:57.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:57 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:57.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:57 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:31:59.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:31:59 smithi055 ceph-mon[24757]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:59.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:31:59 smithi055 ceph-mon[29498]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:31:59.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:31:59 smithi138 ceph-mon[25673]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:01.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:01 smithi055 ceph-mon[24757]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:01.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:01 smithi055 ceph-mon[29498]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:01.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:01 smithi138 ceph-mon[25673]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:04.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:04 smithi055 ceph-mon[24757]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:04.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:04 smithi055 ceph-mon[29498]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:04.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:04 smithi138 ceph-mon[25673]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:06.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:32:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:32:06.434 INFO:teuthology.orchestra.run.smithi055.stdout:167503724567 2024-05-20T22:32:06.434 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-05-20T22:32:06.462 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:06 smithi055 ceph-mon[24757]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:06.462 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:06 smithi055 ceph-mon[29498]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:06.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:06 smithi138 ceph-mon[25673]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:07.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:32:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:07] "GET /metrics HTTP/1.1" 200 191491 "" "Prometheus/2.33.4" 2024-05-20T22:32:07.639 INFO:teuthology.orchestra.run.smithi055.stdout:201863462929 2024-05-20T22:32:07.639 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-05-20T22:32:08.376 INFO:teuthology.orchestra.run.smithi055.stdout:77309411365 2024-05-20T22:32:08.376 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-05-20T22:32:08.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:08 smithi055 ceph-mon[24757]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:08.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:08 smithi055 ceph-mon[29498]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:08.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:08 smithi138 ceph-mon[25673]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:08.768 INFO:teuthology.orchestra.run.smithi055.stdout:236223201294 2024-05-20T22:32:08.769 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-05-20T22:32:09.560 INFO:teuthology.orchestra.run.smithi055.stdout:55834574890 2024-05-20T22:32:09.560 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-05-20T22:32:10.221 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:10.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:10 smithi055 ceph-mon[29498]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:10.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:10 smithi055 ceph-mon[24757]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:10.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:10 smithi138 ceph-mon[25673]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:10.708 INFO:teuthology.orchestra.run.smithi055.stdout:111669149728 2024-05-20T22:32:10.709 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-05-20T22:32:11.174 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:11.654 INFO:teuthology.orchestra.run.smithi055.stdout:34359738416 2024-05-20T22:32:11.655 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-05-20T22:32:11.830 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:12.260 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:12.260 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:12.260 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:12 smithi055 ceph-mon[24757]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:12.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:12 smithi055 ceph-mon[29498]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:12.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:12 smithi138 ceph-mon[25673]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:12.734 INFO:teuthology.orchestra.run.smithi055.stdout:133143986203 2024-05-20T22:32:12.734 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-05-20T22:32:14.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:14 smithi055 ceph-mon[24757]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:14.492 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:14 smithi055 ceph-mon[29498]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:14.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:14 smithi138 ceph-mon[25673]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:16.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:32:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:32:16.518 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:16 smithi055 ceph-mon[24757]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:16.518 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:16 smithi055 ceph-mon[29498]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:16.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:16 smithi138 ceph-mon[25673]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:17.490 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:32:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:17] "GET /metrics HTTP/1.1" 200 191487 "" "Prometheus/2.33.4" 2024-05-20T22:32:18.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:18 smithi055 ceph-mon[24757]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:18.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:18 smithi055 ceph-mon[29498]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:18.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:18 smithi138 ceph-mon[25673]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:19.433 INFO:teuthology.orchestra.run.smithi055.stdout:236223201297 2024-05-20T22:32:19.664 INFO:teuthology.orchestra.run.smithi055.stdout:55834574893 2024-05-20T22:32:20.398 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574893 for osd.1 2024-05-20T22:32:20.398 DEBUG:teuthology.parallel:result is None 2024-05-20T22:32:20.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:20 smithi055 ceph-mon[24757]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:20.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:20 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2434156846' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-20T22:32:20.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:20 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2163037693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-20T22:32:20.524 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:20 smithi055 ceph-mon[29498]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:20.524 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:20 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2434156846' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-20T22:32:20.524 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:20 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2163037693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-20T22:32:20.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:20 smithi138 ceph-mon[25673]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:20.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:20 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2434156846' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-20T22:32:20.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:20 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2163037693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-20T22:32:20.827 INFO:teuthology.orchestra.run.smithi055.stdout:77309411368 2024-05-20T22:32:21.018 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:21.208 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:21.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:21 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2074942770' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-20T22:32:21.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:21 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2074942770' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-20T22:32:21.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:21 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2074942770' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-20T22:32:21.684 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411368 for osd.2 2024-05-20T22:32:21.684 DEBUG:teuthology.parallel:result is None 2024-05-20T22:32:22.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:22 smithi138 ceph-mon[25673]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:22.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:22 smithi055 ceph-mon[24757]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:22.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:22 smithi055 ceph-mon[29498]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:22.843 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201297 for osd.7 2024-05-20T22:32:22.844 DEBUG:teuthology.parallel:result is None 2024-05-20T22:32:23.460 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:23 smithi055 ceph-mon[24757]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:23.460 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:23 smithi055 ceph-mon[29498]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:23.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:23 smithi138 ceph-mon[25673]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:23.679 INFO:teuthology.orchestra.run.smithi055.stdout:201863462933 2024-05-20T22:32:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:24 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2001742847' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-20T22:32:24.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:24 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2001742847' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-20T22:32:24.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:24 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2001742847' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-20T22:32:24.673 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462929 got 201863462933 for osd.6 2024-05-20T22:32:24.673 DEBUG:teuthology.parallel:result is None 2024-05-20T22:32:25.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:25 smithi138 ceph-mon[25673]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:25.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:25 smithi055 ceph-mon[24757]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:25.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:25 smithi055 ceph-mon[29498]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:26.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:32:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:32:26.560 INFO:teuthology.orchestra.run.smithi055.stdout:111669149732 2024-05-20T22:32:26.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:26 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:32:26.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:26 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:32:26.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:26 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:32:26.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:26 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:32:26.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:26 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:32:26.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:26 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:32:26.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:26 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:32:26.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:26 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:32:26.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:26 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:32:26.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:26 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:32:26.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:26 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:32:26.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:26 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:32:27.225 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149732 for osd.3 2024-05-20T22:32:27.225 DEBUG:teuthology.parallel:result is None 2024-05-20T22:32:27.425 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:32:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:27] "GET /metrics HTTP/1.1" 200 191480 "" "Prometheus/2.33.4" 2024-05-20T22:32:27.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:27 smithi055 ceph-mon[24757]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:27.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:27 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2325980705' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-20T22:32:27.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:27 smithi055 ceph-mon[29498]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:27.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:27 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2325980705' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-20T22:32:27.849 INFO:teuthology.orchestra.run.smithi055.stdout:133143986206 2024-05-20T22:32:27.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:27 smithi138 ceph-mon[25673]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:27.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:27 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2325980705' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-20T22:32:28.063 INFO:teuthology.orchestra.run.smithi055.stdout:167503724571 2024-05-20T22:32:28.589 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986203 got 133143986206 for osd.4 2024-05-20T22:32:28.589 DEBUG:teuthology.parallel:result is None 2024-05-20T22:32:28.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:28 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2397967995' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-20T22:32:28.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:28 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3743453544' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-20T22:32:28.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:28 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2397967995' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-20T22:32:28.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:28 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3743453544' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-20T22:32:28.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:28 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2397967995' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-20T22:32:28.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:28 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3743453544' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-20T22:32:29.766 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724567 got 167503724571 for osd.5 2024-05-20T22:32:29.766 DEBUG:teuthology.parallel:result is None 2024-05-20T22:32:30.183 INFO:teuthology.orchestra.run.smithi055.stdout:34359738420 2024-05-20T22:32:30.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:29 smithi055 ceph-mon[24757]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:30.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:30.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:30.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:29 smithi055 ceph-mon[29498]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:30.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:30.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:30.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:29 smithi138 ceph-mon[25673]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:30.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:30.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:30.945 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738420 for osd.0 2024-05-20T22:32:30.945 DEBUG:teuthology.parallel:result is None 2024-05-20T22:32:30.945 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-20T22:32:30.946 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T22:32:31.167 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:31.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:30 smithi055 ceph-mon[24757]: Deploying daemon alertmanager.a on smithi055 2024-05-20T22:32:31.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:30 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1489029528' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-20T22:32:31.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:30 smithi055 ceph-mon[29498]: Deploying daemon alertmanager.a on smithi055 2024-05-20T22:32:31.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:30 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1489029528' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-20T22:32:31.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:30 smithi138 ceph-mon[25673]: Deploying daemon alertmanager.a on smithi055 2024-05-20T22:32:31.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:30 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1489029528' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-20T22:32:31.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:31 smithi055 ceph-mon[24757]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:31.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:31 smithi055 ceph-mon[29498]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:32.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:31 smithi138 ceph-mon[25673]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:34.175 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:32:34.175 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2024-05-20T22:32:34.294 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:34 smithi055 ceph-mon[24757]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:34.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:34 smithi055 ceph-mon[29498]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:34.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:34 smithi138 ceph-mon[25673]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:35.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:35 smithi055 ceph-mon[24757]: from='client.24502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:32:35.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:35 smithi055 ceph-mon[29498]: from='client.24502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:32:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:35 smithi138 ceph-mon[25673]: from='client.24502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:32:36.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:32:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:32:36.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:36 smithi055 ceph-mon[24757]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:36.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:36 smithi055 ceph-mon[29498]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:36.534 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2024-05-20T22:32:32.339026+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48968,"kb_used_data":6152,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912248,"statfs":{"total":767960285184,"available":767910141952,"internally_reserved":0,"allocated":6299648,"data_stored":3511025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002080"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-05-20T22:31:26.411189+0000","last_change":"2024-05-20T22:31:10.214662+0000","last_active":"2024-05-20T22:31:26.411189+0000","last_peered":"2024-05-20T22:31:26.411189+0000","last_clean":"2024-05-20T22:31:26.411189+0000","last_became_active":"2024-05-20T22:31:10.214109+0000","last_became_peered":"2024-05-20T22:31:10.214109+0000","last_unstale":"2024-05-20T22:31:26.411189+0000","last_undegraded":"2024-05-20T22:31:26.411189+0000","last_fullsized":"2024-05-20T22:31:26.411189+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T22:29:13.339364+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T22:29:13.339364+0000","last_clean_scrub_stamp":"2024-05-20T22:29:13.339364+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T10:07:25.829739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.622}]},{"osd":1,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.58799999999999997}]},{"osd":2,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.56000000000000005}]},{"osd":3,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.59999999999999998}]},{"osd":4,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.498}]},{"osd":5,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.42499999999999999}]},{"osd":6,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.54100000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6160,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738992,"statfs":{"total":95995035648,"available":95988727808,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.03,"5min":1.03,"15min":1.03},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":11.331,"5min":11.331,"15min":11.331},"last":0.54600000000000004}]},{"osd":1,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.53600000000000003}]},{"osd":2,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.497},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.48399999999999999}]},{"osd":3,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.59399999999999997}]},{"osd":4,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.442},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.61199999999999999}]},{"osd":7,"last update":"Mon May 20 22:32:07 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.61899999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:31:51 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.38700000000000001,"15min":0.36499999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.63,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.375,"15min":0.35199999999999998},"min":{"1min":0.29599999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.59999999999999998,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.46400000000000002}]},{"osd":2,"last update":"Mon May 20 22:32:14 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.39800000000000002,"15min":0.38300000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.42399999999999999,"15min":0.40899999999999997},"min":{"1min":0.32900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.64800000000000002,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.5}]},{"osd":3,"last update":"Mon May 20 22:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.48499999999999999,"15min":0.48099999999999998},"min":{"1min":0.40699999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.45600000000000002,"15min":0.45400000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.57199999999999995,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.56999999999999995}]},{"osd":4,"last update":"Mon May 20 22:31:59 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.63300000000000001,"15min":0.63900000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.96299999999999997,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.63400000000000001,"15min":0.63200000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.81200000000000006,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.90500000000000003}]},{"osd":5,"last update":"Mon May 20 22:32:23 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.624,"15min":0.61699999999999999},"min":{"1min":0.47999999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.63400000000000001,"15min":0.628},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.84399999999999997}]},{"osd":6,"last update":"Mon May 20 22:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.76200000000000001}]},{"osd":7,"last update":"Mon May 20 22:32:10 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.74299999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6924,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738228,"statfs":{"total":95995035648,"available":95987945472,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon May 20 22:31:56 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36199999999999999,"15min":0.34300000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.379,"15min":0.35699999999999998},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52600000000000002,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.375}]},{"osd":2,"last update":"Mon May 20 22:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.40799999999999997,"15min":0.40200000000000002},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.33100000000000002}]},{"osd":3,"last update":"Mon May 20 22:31:37 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.46000000000000002,"15min":0.46400000000000002},"min":{"1min":0.35099999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.58699999999999997,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.31900000000000001}]},{"osd":4,"last update":"Mon May 20 22:32:02 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.59299999999999997,"15min":0.59899999999999998},"min":{"1min":0.43099999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.751,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.58099999999999996,"15min":0.58899999999999997},"min":{"1min":0.41199999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.53500000000000003}]},{"osd":5,"last update":"Mon May 20 22:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.45500000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.442},{"interface":"front","average":{"1min":0.62,"5min":0.59799999999999998,"15min":0.59399999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.495}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.432}]},{"osd":7,"last update":"Mon May 20 22:32:07 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.52800000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.39600000000000002,"15min":0.40000000000000002},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41699999999999998,"15min":0.40500000000000003},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.395,"5min":1.395,"15min":1.395},"last":0.40999999999999998}]},{"osd":1,"last update":"Mon May 20 22:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.25700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45400000000000001,"15min":0.44400000000000001},"min":{"1min":0.26800000000000002,"5min":0.246,"15min":0.246},"max":{"1min":1.4379999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.46700000000000003}]},{"osd":3,"last update":"Mon May 20 22:31:44 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.46500000000000002,"15min":0.46200000000000002},"min":{"1min":0.36799999999999999,"5min":0.315,"15min":0.315},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.47899999999999998,"15min":0.47399999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.48999999999999999}]},{"osd":4,"last update":"Mon May 20 22:32:04 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.623,"15min":0.61299999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.7450000000000001,"5min":1.7450000000000001,"15min":1.7450000000000001},"last":0.76300000000000001}]},{"osd":5,"last update":"Mon May 20 22:32:25 2024","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.63800000000000001,"15min":0.63},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.60599999999999998,"15min":0.61099999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83199999999999996,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.64300000000000002}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.625},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.747}]},{"osd":7,"last update":"Mon May 20 22:32:07 2024","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.78,"5min":1.78,"15min":1.78},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.65800000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:31:42 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.437,"15min":0.435},"min":{"1min":0.29699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.40799999999999997}]},{"osd":1,"last update":"Mon May 20 22:31:42 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.45500000000000002,"15min":0.46100000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.54300000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.45800000000000002,"15min":0.46300000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55800000000000005,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon May 20 22:31:42 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.47399999999999998,"15min":0.47799999999999998},"min":{"1min":0.312,"5min":0.307,"15min":0.307},"max":{"1min":0.63900000000000001,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.46999999999999997,"15min":0.47399999999999998},"min":{"1min":0.308,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.48899999999999999}]},{"osd":4,"last update":"Mon May 20 22:32:01 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.57399999999999995,"15min":0.56399999999999995},"min":{"1min":0.46800000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.57499999999999996,"15min":0.56999999999999995},"min":{"1min":0.41499999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.60199999999999998}]},{"osd":5,"last update":"Mon May 20 22:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.42699999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.623},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.60799999999999998,"15min":0.60399999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.56200000000000006}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.64900000000000002}]},{"osd":7,"last update":"Mon May 20 22:32:10 2024","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.63900000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:32:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.56399999999999995,"15min":0.56899999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71099999999999997,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.433},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55200000000000005,"15min":0.55300000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.73199999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.46000000000000002}]},{"osd":1,"last update":"Mon May 20 22:32:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.56000000000000005,"15min":0.56499999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76400000000000001,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.39200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.75,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon May 20 22:32:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.35899999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.55500000000000005,"15min":0.55900000000000005},"min":{"1min":0.39000000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69699999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.56699999999999995}]},{"osd":3,"last update":"Mon May 20 22:32:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.41099999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77500000000000002,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.495},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.56499999999999995,"15min":0.57299999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.73899999999999999,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.48799999999999999}]},{"osd":5,"last update":"Mon May 20 22:32:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.39800000000000002,"15min":0.39300000000000002},"min":{"1min":0.27400000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.41999999999999998,"15min":0.41399999999999998},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.22},"max":{"1min":0.60499999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.58799999999999997}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.47699999999999998}]},{"osd":7,"last update":"Mon May 20 22:32:08 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.51600000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":240130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.52900000000000003,"15min":0.52500000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.55700000000000005}]},{"osd":1,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.753,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.52300000000000002,"15min":0.51700000000000002},"min":{"1min":0.36799999999999999,"5min":0.312,"15min":0.312},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.51400000000000001}]},{"osd":2,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.50700000000000001,"15min":0.502},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.53000000000000003,"15min":0.52300000000000002},"min":{"1min":0.48099999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.68200000000000005}]},{"osd":3,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56000000000000005,"15min":0.55300000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.56699999999999995,"15min":0.56100000000000005},"min":{"1min":0.436,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.57299999999999995}]},{"osd":4,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.41599999999999998,"15min":0.40799999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.41399999999999998,"15min":0.40400000000000003},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.59999999999999998}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.70499999999999996}]},{"osd":7,"last update":"Mon May 20 22:32:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.65500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T22:32:36.536 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T22:32:36.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:36 smithi138 ceph-mon[25673]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:37.292 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:32:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:37] "GET /metrics HTTP/1.1" 200 191480 "" "Prometheus/2.33.4" 2024-05-20T22:32:37.434 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:38.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:38 smithi055 ceph-mon[29498]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:38.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:38 smithi055 ceph-mon[24757]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:38.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:38 smithi138 ceph-mon[25673]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:39.525 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:32:39.526 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2024-05-20T22:32:39.558 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:39 smithi055 podman[47843]: 2024-05-20 22:32:39.454165173 +0000 UTC m=+0.024686353 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-20T22:32:39.558 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:39 smithi055 podman[47843]: 2024-05-20 22:32:39.563282035 +0000 UTC m=+0.133803183 volume create 9a36ef3dba7e0a0172eb0ead737e14522400b6cc9175f7704c4bc451e3971858 2024-05-20T22:32:39.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:39 smithi055 podman[47843]: 2024-05-20 22:32:39.664006258 +0000 UTC m=+0.234527408 container create 3eb20538cfe349b4ede51e04a35696e8512e2716ee14a14656959589ebf0c9cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:32:40.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:40 smithi055 ceph-mon[24757]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:40.395 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:40 smithi055 ceph-mon[29498]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:40.420 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-05-20T22:32:38.340103+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48968,"kb_used_data":6152,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912248,"statfs":{"total":767960285184,"available":767910141952,"internally_reserved":0,"allocated":6299648,"data_stored":3511025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002164"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-05-20T22:31:26.411189+0000","last_change":"2024-05-20T22:31:10.214662+0000","last_active":"2024-05-20T22:31:26.411189+0000","last_peered":"2024-05-20T22:31:26.411189+0000","last_clean":"2024-05-20T22:31:26.411189+0000","last_became_active":"2024-05-20T22:31:10.214109+0000","last_became_peered":"2024-05-20T22:31:10.214109+0000","last_unstale":"2024-05-20T22:31:26.411189+0000","last_undegraded":"2024-05-20T22:31:26.411189+0000","last_fullsized":"2024-05-20T22:31:26.411189+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T22:29:13.339364+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T22:29:13.339364+0000","last_clean_scrub_stamp":"2024-05-20T22:29:13.339364+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T10:07:25.829739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.63600000000000001}]},{"osd":1,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.73399999999999999}]},{"osd":2,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.61699999999999999}]},{"osd":3,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.59699999999999998}]},{"osd":4,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon May 20 22:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.623}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6160,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738992,"statfs":{"total":95995035648,"available":95988727808,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.03,"5min":1.03,"15min":1.03},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":11.331,"5min":11.331,"15min":11.331},"last":0.45600000000000002}]},{"osd":1,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.42999999999999999}]},{"osd":2,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.498}]},{"osd":3,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.55400000000000005}]},{"osd":4,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.375},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.54000000000000004}]},{"osd":5,"last update":"Mon May 20 22:31:46 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.57599999999999996}]},{"osd":7,"last update":"Mon May 20 22:32:07 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.442},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.52900000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:31:51 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.38700000000000001,"15min":0.36499999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.63,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.375,"15min":0.35199999999999998},"min":{"1min":0.29599999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.59999999999999998,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.52600000000000002}]},{"osd":2,"last update":"Mon May 20 22:32:14 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.39800000000000002,"15min":0.38300000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.42399999999999999,"15min":0.40899999999999997},"min":{"1min":0.32900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.64800000000000002,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon May 20 22:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.48499999999999999,"15min":0.48099999999999998},"min":{"1min":0.40699999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.45600000000000002,"15min":0.45400000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.57199999999999995,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.53700000000000003}]},{"osd":4,"last update":"Mon May 20 22:31:59 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.63300000000000001,"15min":0.63900000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.96299999999999997,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.63400000000000001,"15min":0.63200000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.81200000000000006,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.79600000000000004}]},{"osd":5,"last update":"Mon May 20 22:32:23 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.624,"15min":0.61699999999999999},"min":{"1min":0.47999999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.63400000000000001,"15min":0.628},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.90600000000000003}]},{"osd":6,"last update":"Mon May 20 22:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.93999999999999995}]},{"osd":7,"last update":"Mon May 20 22:32:10 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.92000000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6924,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738228,"statfs":{"total":95995035648,"available":95987945472,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon May 20 22:31:56 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36199999999999999,"15min":0.34300000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.379,"15min":0.35699999999999998},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52600000000000002,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.44700000000000001}]},{"osd":2,"last update":"Mon May 20 22:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.40799999999999997,"15min":0.40200000000000002},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.48599999999999999}]},{"osd":3,"last update":"Mon May 20 22:31:37 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.436,"5min":0.46000000000000002,"15min":0.46400000000000002},"min":{"1min":0.35099999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.58699999999999997,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.52900000000000003}]},{"osd":4,"last update":"Mon May 20 22:32:02 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.59299999999999997,"15min":0.59899999999999998},"min":{"1min":0.43099999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.751,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.58099999999999996,"15min":0.58899999999999997},"min":{"1min":0.41199999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.64200000000000002}]},{"osd":5,"last update":"Mon May 20 22:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.45500000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.59799999999999998,"15min":0.59399999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.77700000000000002}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.68300000000000005}]},{"osd":7,"last update":"Mon May 20 22:32:07 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.81999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.39600000000000002,"15min":0.40000000000000002},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41699999999999998,"15min":0.40500000000000003},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.395,"5min":1.395,"15min":1.395},"last":0.505}]},{"osd":1,"last update":"Mon May 20 22:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.25700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45400000000000001,"15min":0.44400000000000001},"min":{"1min":0.26800000000000002,"5min":0.246,"15min":0.246},"max":{"1min":1.4379999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon May 20 22:31:44 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.46500000000000002,"15min":0.46200000000000002},"min":{"1min":0.36799999999999999,"5min":0.315,"15min":0.315},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.47899999999999998,"15min":0.47399999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.60099999999999998}]},{"osd":4,"last update":"Mon May 20 22:32:04 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.623,"15min":0.61299999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.7450000000000001,"5min":1.7450000000000001,"15min":1.7450000000000001},"last":0.83499999999999996}]},{"osd":5,"last update":"Mon May 20 22:32:25 2024","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.63800000000000001,"15min":0.63},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.60599999999999998,"15min":0.61099999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83199999999999996,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.752}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.87},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.72699999999999998}]},{"osd":7,"last update":"Mon May 20 22:32:07 2024","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.78,"5min":1.78,"15min":1.78},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.85399999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:31:42 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.437,"15min":0.435},"min":{"1min":0.29699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.24299999999999999}]},{"osd":1,"last update":"Mon May 20 22:31:42 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.45500000000000002,"15min":0.46100000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.54300000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.308},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.45800000000000002,"15min":0.46300000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55800000000000005,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Mon May 20 22:31:42 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.47399999999999998,"15min":0.47799999999999998},"min":{"1min":0.312,"5min":0.307,"15min":0.307},"max":{"1min":0.63900000000000001,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.46999999999999997,"15min":0.47399999999999998},"min":{"1min":0.308,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.33700000000000002}]},{"osd":4,"last update":"Mon May 20 22:32:01 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.57399999999999995,"15min":0.56399999999999995},"min":{"1min":0.46800000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.57499999999999996,"15min":0.56999999999999995},"min":{"1min":0.41499999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.57099999999999995}]},{"osd":5,"last update":"Mon May 20 22:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.42699999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.60799999999999998,"15min":0.60399999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.63300000000000001}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.624}]},{"osd":7,"last update":"Mon May 20 22:32:10 2024","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.66300000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:32:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.56399999999999995,"15min":0.56899999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71099999999999997,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.433},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55200000000000005,"15min":0.55300000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.73199999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.46000000000000002}]},{"osd":1,"last update":"Mon May 20 22:32:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.56000000000000005,"15min":0.56499999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76400000000000001,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.39200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.75,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon May 20 22:32:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.35899999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.55500000000000005,"15min":0.55900000000000005},"min":{"1min":0.39000000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69699999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.56699999999999995}]},{"osd":3,"last update":"Mon May 20 22:32:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.41099999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77500000000000002,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.495},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.56499999999999995,"15min":0.57299999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.73899999999999999,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.48799999999999999}]},{"osd":5,"last update":"Mon May 20 22:32:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.39800000000000002,"15min":0.39300000000000002},"min":{"1min":0.27400000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.41999999999999998,"15min":0.41399999999999998},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.22},"max":{"1min":0.60499999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.58799999999999997}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.47699999999999998}]},{"osd":7,"last update":"Mon May 20 22:32:08 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.51600000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":240130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.52900000000000003,"15min":0.52500000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.67400000000000004}]},{"osd":1,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.753,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.52300000000000002,"15min":0.51700000000000002},"min":{"1min":0.36799999999999999,"5min":0.312,"15min":0.312},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.59099999999999997}]},{"osd":2,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.50700000000000001,"15min":0.502},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.53000000000000003,"15min":0.52300000000000002},"min":{"1min":0.48099999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.70599999999999996}]},{"osd":3,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56000000000000005,"15min":0.55300000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.56699999999999995,"15min":0.56100000000000005},"min":{"1min":0.436,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.75600000000000001}]},{"osd":4,"last update":"Mon May 20 22:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.41599999999999998,"15min":0.40799999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.41399999999999998,"15min":0.40400000000000003},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.60699999999999998}]},{"osd":6,"last update":"Mon May 20 22:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.56399999999999995}]},{"osd":7,"last update":"Mon May 20 22:32:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.625},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.63600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T22:32:40.421 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-20T22:32:40.421 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-05-20T22:32:40.421 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-05-20T22:32:40.422 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- ceph health --format=json 2024-05-20T22:32:40.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:40 smithi138 ceph-mon[25673]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:40.830 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T22:32:40.955 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 podman[47843]: 2024-05-20 22:32:40.702683572 +0000 UTC m=+1.273204717 container init 3eb20538cfe349b4ede51e04a35696e8512e2716ee14a14656959589ebf0c9cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:32:40.955 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 podman[47843]: 2024-05-20 22:32:40.706133273 +0000 UTC m=+1.276654421 container start 3eb20538cfe349b4ede51e04a35696e8512e2716ee14a14656959589ebf0c9cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:32:40.955 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:40.717Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-20T22:32:40.955 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:40.717Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-20T22:32:40.955 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:40.718Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.55 port=9094 2024-05-20T22:32:40.956 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:40.720Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-20T22:32:40.956 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:40.745Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-20T22:32:40.956 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:40.745Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-20T22:32:40.956 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:40.747Z caller=main.go:518 msg=Listening address=:9093 2024-05-20T22:32:40.956 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:40.747Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-20T22:32:40.956 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 bash[47843]: 3eb20538cfe349b4ede51e04a35696e8512e2716ee14a14656959589ebf0c9cb 2024-05-20T22:32:40.956 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:40 smithi055 systemd[1]: Started Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:32:41.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:41 smithi055 ceph-mon[24757]: from='client.24508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:32:41.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:41 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:41.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:41 smithi055 ceph-mon[29498]: from='client.24508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:32:41.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:41 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:41.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:41 smithi138 ceph-mon[25673]: from='client.24508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:32:41.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:41 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.378 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[24757]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:42.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:32:42.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:32:42.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[24757]: Deploying daemon grafana.a on smithi138 2024-05-20T22:32:42.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[29498]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:42.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.380 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.380 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:32:42.380 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:32:42.380 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.380 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[29498]: Deploying daemon grafana.a on smithi138 2024-05-20T22:32:42.380 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:42 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:42 smithi138 ceph-mon[25673]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:42.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:42 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:42 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:42 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:32:42.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:42 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:32:42.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:42 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:42 smithi138 ceph-mon[25673]: Deploying daemon grafana.a on smithi138 2024-05-20T22:32:42.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:42 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:42.845 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:32:42.845 INFO:teuthology.orchestra.run.smithi055.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-05-20T22:32:42.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:42.720Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000484753s 2024-05-20T22:32:43.478 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:43 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/146298294' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-20T22:32:43.478 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:43 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/146298294' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-20T22:32:43.479 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-05-20T22:32:43.479 INFO:tasks.cephadm:Setup complete, yielding 2024-05-20T22:32:43.480 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T22:32:43.489 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi055.front.sepia.ceph.com 2024-05-20T22:32:43.490 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-05-20T22:32:43.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:43 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/146298294' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-20T22:32:44.443 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:44 smithi055 ceph-mon[24757]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:44.444 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:44 smithi055 ceph-mon[29498]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:44.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:44 smithi138 ceph-mon[25673]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:45.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:45 smithi055 ceph-mon[24757]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:45.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:45 smithi055 ceph-mon[29498]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:46.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:45 smithi138 ceph-mon[25673]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:46.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:32:45 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:32:46.692 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:32:46.692 INFO:teuthology.orchestra.run.smithi055.stdout: "id": "b8afa262-078e-4a92-9a54-059cd22c53f1", 2024-05-20T22:32:46.692 INFO:teuthology.orchestra.run.smithi055.stdout: "name": "r", 2024-05-20T22:32:46.692 INFO:teuthology.orchestra.run.smithi055.stdout: "current_period": "7f73aa5a-0d61-44d1-8105-d0930b40a807", 2024-05-20T22:32:46.692 INFO:teuthology.orchestra.run.smithi055.stdout: "epoch": 1 2024-05-20T22:32:46.692 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:32:46.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:46 smithi055 ceph-mon[24757]: osdmap e61: 8 total, 8 up, 8 in 2024-05-20T22:32:46.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:46 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1958172924' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-20T22:32:46.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:46 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1958172924' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-20T22:32:46.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:46 smithi055 ceph-mon[24757]: osdmap e62: 8 total, 8 up, 8 in 2024-05-20T22:32:46.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:46 smithi055 ceph-mon[29498]: osdmap e61: 8 total, 8 up, 8 in 2024-05-20T22:32:46.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:46 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1958172924' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-20T22:32:46.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:46 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1958172924' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-20T22:32:46.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:46 smithi055 ceph-mon[29498]: osdmap e62: 8 total, 8 up, 8 in 2024-05-20T22:32:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:46 smithi138 ceph-mon[25673]: osdmap e61: 8 total, 8 up, 8 in 2024-05-20T22:32:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:46 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1958172924' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-20T22:32:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:46 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1958172924' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-20T22:32:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:46 smithi138 ceph-mon[25673]: osdmap e62: 8 total, 8 up, 8 in 2024-05-20T22:32:47.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:32:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:47] "GET /metrics HTTP/1.1" 200 191485 "" "Prometheus/2.33.4" 2024-05-20T22:32:47.549 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-05-20T22:32:48.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:48 smithi055 ceph-mon[29498]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:48.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:48 smithi055 ceph-mon[29498]: osdmap e63: 8 total, 8 up, 8 in 2024-05-20T22:32:48.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:48 smithi055 ceph-mon[24757]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:48.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:48 smithi055 ceph-mon[24757]: osdmap e63: 8 total, 8 up, 8 in 2024-05-20T22:32:48.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:48 smithi138 ceph-mon[25673]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:32:48.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:48 smithi138 ceph-mon[25673]: osdmap e63: 8 total, 8 up, 8 in 2024-05-20T22:32:49.362 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "id": "e72c7d0d-20be-4681-b89e-6add7b1be9d6", 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "name": "default", 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "api_name": "default", 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "is_master": "true", 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "endpoints": [], 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "hostnames": [], 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "hostnames_s3website": [], 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "master_zone": "", 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "zones": [], 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "placement_targets": [], 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "default_placement": "", 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "realm_id": "b8afa262-078e-4a92-9a54-059cd22c53f1", 2024-05-20T22:32:49.363 INFO:teuthology.orchestra.run.smithi055.stdout: "sync_policy": { 2024-05-20T22:32:49.364 INFO:teuthology.orchestra.run.smithi055.stdout: "groups": [] 2024-05-20T22:32:49.364 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:32:49.364 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:32:50.056 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-05-20T22:32:50.375 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:50 smithi055 ceph-mon[24757]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-20T22:32:50.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:50 smithi055 ceph-mon[29498]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-20T22:32:50.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:50 smithi138 ceph-mon[25673]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-20T22:32:50.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:32:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:32:50.721Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.001475281s 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "id": "9e6ed268-ecdc-4e24-ae7d-711f622a25df", 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "name": "z", 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "domain_root": "z.rgw.meta:root", 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "control_pool": "z.rgw.control", 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "gc_pool": "z.rgw.log:gc", 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "lc_pool": "z.rgw.log:lc", 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "log_pool": "z.rgw.log", 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-05-20T22:32:52.028 INFO:teuthology.orchestra.run.smithi055.stdout: "roles_pool": "z.rgw.meta:roles", 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "otp_pool": "z.rgw.otp", 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "system_key": { 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "access_key": "", 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "secret_key": "" 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "placement_pools": [ 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2024-05-20T22:32:52.029 INFO:teuthology.orchestra.run.smithi055.stdout: "key": "default-placement", 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: "val": { 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: "index_pool": "z.rgw.buckets.index", 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: "storage_classes": { 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: "STANDARD": { 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: "data_pool": "z.rgw.buckets.data" 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: "index_type": 0 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:32:52.030 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:32:52.031 INFO:teuthology.orchestra.run.smithi055.stdout: "realm_id": "b8afa262-078e-4a92-9a54-059cd22c53f1", 2024-05-20T22:32:52.031 INFO:teuthology.orchestra.run.smithi055.stdout: "notif_pool": "z.rgw.log:notif" 2024-05-20T22:32:52.031 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:32:52.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:52 smithi055 ceph-mon[24757]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-20T22:32:52.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:52 smithi055 ceph-mon[29498]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-20T22:32:52.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:52 smithi138 ceph-mon[25673]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-05-20T22:32:52.714 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-05-20T22:32:53.529 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 systemd[1]: Starting Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:32:53.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 podman[41669]: 2024-05-20 22:32:53.436327474 +0000 UTC m=+0.016297588 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-05-20T22:32:53.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 podman[41669]: 2024-05-20 22:32:53.547136944 +0000 UTC m=+0.127107079 container create 93d02684b943a111c3796a5b2362ee0edf510b2f99ba720844178f2d0cdaf198 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, name=ubi8, io.openshift.tags=base rhel8, release=236.1648460182, com.redhat.component=ubi8-container, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, io.buildah.version=1.24.2, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5) 2024-05-20T22:32:53.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 podman[41669]: 2024-05-20 22:32:53.767205672 +0000 UTC m=+0.347175785 container init 93d02684b943a111c3796a5b2362ee0edf510b2f99ba720844178f2d0cdaf198 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, version=8.5, io.openshift.expose-services=, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, vendor=Red Hat, Inc., name=ubi8, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-05-20T22:32:53.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 podman[41669]: 2024-05-20 22:32:53.770163811 +0000 UTC m=+0.350133925 container start 93d02684b943a111c3796a5b2362ee0edf510b2f99ba720844178f2d0cdaf198 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, description=Ceph Grafana Container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., version=8.5, io.openshift.expose-services=, io.buildah.version=1.24.2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, distribution-scope=public, release=236.1648460182, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, vcs-type=git) 2024-05-20T22:32:54.044 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-05-20T22:32:54.044 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-05-20T22:32:54.044 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-05-20T22:32:54.045 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="App mode production" logger=settings 2024-05-20T22:32:54.046 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-05-20T22:32:54.046 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-20T22:32:54.046 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-05-20T22:32:54.046 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-05-20T22:32:54.046 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-05-20T22:32:54.047 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-05-20T22:32:54.047 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-05-20T22:32:54.047 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-05-20T22:32:54.047 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-05-20T22:32:54.048 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-05-20T22:32:54.049 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 bash[41669]: 93d02684b943a111c3796a5b2362ee0edf510b2f99ba720844178f2d0cdaf198 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 systemd[1]: Started Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-05-20T22:32:54.050 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-05-20T22:32:54.051 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-05-20T22:32:54.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-05-20T22:32:54.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-05-20T22:32:54.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-05-20T22:32:54.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-05-20T22:32:54.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-05-20T22:32:54.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-05-20T22:32:54.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-05-20T22:32:54.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-05-20T22:32:54.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-05-20T22:32:54.054 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-05-20T22:32:54.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-05-20T22:32:54.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-05-20T22:32:54.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-05-20T22:32:54.058 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-05-20T22:32:54.059 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-05-20T22:32:54.060 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-05-20T22:32:54.060 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-05-20T22:32:54.060 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-05-20T22:32:54.060 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-05-20T22:32:54.060 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-05-20T22:32:54.061 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-05-20T22:32:54.061 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-05-20T22:32:54.061 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-05-20T22:32:54.061 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-05-20T22:32:54.061 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-05-20T22:32:54.061 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-05-20T22:32:54.061 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-05-20T22:32:54.061 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-05-20T22:32:54.062 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-05-20T22:32:54.063 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-05-20T22:32:54.064 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-05-20T22:32:54.065 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-05-20T22:32:54.066 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-05-20T22:32:54.067 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-05-20T22:32:54.301 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:54 smithi138 ceph-mon[25673]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 2.4 KiB/s wr, 4 op/s 2024-05-20T22:32:54.301 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:54 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:54.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:54 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:54.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:54 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:54.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:54 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:32:54.302 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-05-20T22:32:54.302 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-05-20T22:32:54.302 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-05-20T22:32:54.303 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-05-20T22:32:54.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-05-20T22:32:54.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-05-20T22:32:54.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-05-20T22:32:54.307 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-05-20T22:32:54.307 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-05-20T22:32:54.307 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-05-20T22:32:54.308 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-05-20T22:32:54.309 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-05-20T22:32:54.310 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-05-20T22:32:54.311 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-05-20T22:32:54.312 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-05-20T22:32:54.313 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-05-20T22:32:54.314 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-05-20T22:32:54.315 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=487.317312ms 2024-05-20T22:32:54.316 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-05-20T22:32:54.317 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-05-20T22:32:54.317 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-05-20T22:32:54.317 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[24757]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 2.4 KiB/s wr, 4 op/s 2024-05-20T22:32:54.317 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:54.317 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:54.317 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:54.317 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:32:54.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[29498]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 2.4 KiB/s wr, 4 op/s 2024-05-20T22:32:54.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:54.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:54.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:54.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:54 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:32:54.609 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-05-20T22:32:54.609 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-05-20T22:32:54.609 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-05-20T22:32:54.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-05-20T22:32:54.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-20T22:32:54.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-05-20T22:32:54.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-05-20T22:32:54.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:32:54 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:32:54+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-05-20T22:32:56.074 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:32:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:32:56.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:56 smithi138 ceph-mon[25673]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-05-20T22:32:56.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:56 smithi138 ceph-mon[25673]: osdmap e64: 8 total, 8 up, 8 in 2024-05-20T22:32:56.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:56 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3004216733' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-20T22:32:56.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:56 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-20T22:32:56.660 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:56 smithi055 ceph-mon[24757]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-05-20T22:32:56.660 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:56 smithi055 ceph-mon[24757]: osdmap e64: 8 total, 8 up, 8 in 2024-05-20T22:32:56.661 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:56 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3004216733' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-20T22:32:56.661 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:56 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-20T22:32:56.661 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:56 smithi055 ceph-mon[29498]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-05-20T22:32:56.661 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:56 smithi055 ceph-mon[29498]: osdmap e64: 8 total, 8 up, 8 in 2024-05-20T22:32:56.661 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:56 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3004216733' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-20T22:32:56.661 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:56 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-20T22:32:57.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:32:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:32:57] "GET /metrics HTTP/1.1" 200 194254 "" "Prometheus/2.33.4" 2024-05-20T22:32:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-20T22:32:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[29498]: osdmap e65: 8 total, 8 up, 8 in 2024-05-20T22:32:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[29498]: pgmap v53: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 1022 B/s wr, 2 op/s 2024-05-20T22:32:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[29498]: osdmap e66: 8 total, 8 up, 8 in 2024-05-20T22:32:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3004216733' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-20T22:32:57.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-20T22:32:57.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-20T22:32:57.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[24757]: osdmap e65: 8 total, 8 up, 8 in 2024-05-20T22:32:57.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[24757]: pgmap v53: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 1022 B/s wr, 2 op/s 2024-05-20T22:32:57.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:57.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:57.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:57.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[24757]: osdmap e66: 8 total, 8 up, 8 in 2024-05-20T22:32:57.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3004216733' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-20T22:32:57.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:57 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-20T22:32:57.516 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:57 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-20T22:32:57.516 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:57 smithi138 ceph-mon[25673]: osdmap e65: 8 total, 8 up, 8 in 2024-05-20T22:32:57.516 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:57 smithi138 ceph-mon[25673]: pgmap v53: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 1022 B/s wr, 2 op/s 2024-05-20T22:32:57.516 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:57 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:57.516 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:57 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:57.516 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:57 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:57.517 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:57 smithi138 ceph-mon[25673]: osdmap e66: 8 total, 8 up, 8 in 2024-05-20T22:32:57.517 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:57 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3004216733' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-20T22:32:57.517 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:57 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-20T22:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:58 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:58 smithi138 ceph-mon[25673]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-20T22:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:58 smithi138 ceph-mon[25673]: Reconfiguring daemon alertmanager.a on smithi055 2024-05-20T22:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:58 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-20T22:32:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:32:58 smithi138 ceph-mon[25673]: osdmap e67: 8 total, 8 up, 8 in 2024-05-20T22:32:59.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:59.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[24757]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-20T22:32:59.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[24757]: Reconfiguring daemon alertmanager.a on smithi055 2024-05-20T22:32:59.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-20T22:32:59.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[24757]: osdmap e67: 8 total, 8 up, 8 in 2024-05-20T22:32:59.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:32:59.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[29498]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-20T22:32:59.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[29498]: Reconfiguring daemon alertmanager.a on smithi055 2024-05-20T22:32:59.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-20T22:32:59.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:32:58 smithi055 ceph-mon[29498]: osdmap e67: 8 total, 8 up, 8 in 2024-05-20T22:33:00.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:00 smithi138 ceph-mon[25673]: pgmap v56: 97 pgs: 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-05-20T22:33:00.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:00 smithi138 ceph-mon[25673]: osdmap e68: 8 total, 8 up, 8 in 2024-05-20T22:33:00.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:00 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-20T22:33:00.664 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:00 smithi055 ceph-mon[24757]: pgmap v56: 97 pgs: 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-05-20T22:33:00.664 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:00 smithi055 ceph-mon[24757]: osdmap e68: 8 total, 8 up, 8 in 2024-05-20T22:33:00.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:00 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-20T22:33:00.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:00 smithi055 ceph-mon[29498]: pgmap v56: 97 pgs: 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-05-20T22:33:00.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:00 smithi055 ceph-mon[29498]: osdmap e68: 8 total, 8 up, 8 in 2024-05-20T22:33:00.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:00 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-20T22:33:00.960 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:00 smithi055 systemd[1]: Stopping Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:33:00.960 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:00 smithi055 bash[49235]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager.a" found: no such container 2024-05-20T22:33:01.236 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:00 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[47903]: level=info ts=2024-05-20T22:33:00.965Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-20T22:33:01.236 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:00 smithi055 podman[49246]: 2024-05-20 22:33:00.986382589 +0000 UTC m=+0.139612872 container died 3eb20538cfe349b4ede51e04a35696e8512e2716ee14a14656959589ebf0c9cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:01.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:01 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-20T22:33:01.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:01 smithi138 ceph-mon[25673]: osdmap e69: 8 total, 8 up, 8 in 2024-05-20T22:33:01.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:01 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-20T22:33:01.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:01 smithi138 ceph-mon[25673]: pgmap v59: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-05-20T22:33:01.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:01 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-20T22:33:01.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:01 smithi138 ceph-mon[25673]: osdmap e70: 8 total, 8 up, 8 in 2024-05-20T22:33:01.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:01 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-20T22:33:01.731 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:01 smithi055 podman[49246]: 2024-05-20 22:33:01.482805357 +0000 UTC m=+0.636035643 container cleanup 3eb20538cfe349b4ede51e04a35696e8512e2716ee14a14656959589ebf0c9cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:01.731 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:01 smithi055 bash[49246]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a 2024-05-20T22:33:01.731 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:01 smithi055 bash[49272]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager.a" found: no such container 2024-05-20T22:33:01.731 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-20T22:33:01.731 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[24757]: osdmap e69: 8 total, 8 up, 8 in 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[24757]: pgmap v59: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[24757]: osdmap e70: 8 total, 8 up, 8 in 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[29498]: osdmap e69: 8 total, 8 up, 8 in 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[29498]: pgmap v59: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-20T22:33:01.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[29498]: osdmap e70: 8 total, 8 up, 8 in 2024-05-20T22:33:01.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:01 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-20T22:33:01.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:01 smithi055 podman[49258]: 2024-05-20 22:33:01.736442762 +0000 UTC m=+0.740007824 container remove 3eb20538cfe349b4ede51e04a35696e8512e2716ee14a14656959589ebf0c9cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:01.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:01 smithi055 podman[49258]: 2024-05-20 22:33:01.829284418 +0000 UTC m=+0.832849487 volume remove 9a36ef3dba7e0a0172eb0ead737e14522400b6cc9175f7704c4bc451e3971858 2024-05-20T22:33:01.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:01 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-20T22:33:01.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:01 smithi055 systemd[1]: Stopped Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:33:01.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:01 smithi055 systemd[1]: Starting Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:33:02.342 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:02 smithi055 podman[49338]: 2024-05-20 22:33:01.972620547 +0000 UTC m=+0.018717071 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-20T22:33:02.342 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:02 smithi055 podman[49338]: 2024-05-20 22:33:02.080153557 +0000 UTC m=+0.126250068 volume create 6cf369965a506e9f5accc45de02e6f5799be3f5df61b88ad60e0ca0f0ab57f4a 2024-05-20T22:33:02.342 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:02 smithi055 podman[49338]: 2024-05-20 22:33:02.172340455 +0000 UTC m=+0.218436988 container create 37600d6e6b2f10d71c806a5f11df501466ff201f46d0eccdfb577a76fc5ca17d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:02.351 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "id": "20e6e7ab-acfd-4c9e-8e72-66a5a40126ab", 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "epoch": 1, 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "predecessor_uuid": "7f73aa5a-0d61-44d1-8105-d0930b40a807", 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "sync_status": [], 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "period_map": { 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "id": "20e6e7ab-acfd-4c9e-8e72-66a5a40126ab", 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "zonegroups": [ 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "id": "e72c7d0d-20be-4681-b89e-6add7b1be9d6", 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "name": "default", 2024-05-20T22:33:02.352 INFO:teuthology.orchestra.run.smithi055.stdout: "api_name": "default", 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "is_master": "true", 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "endpoints": [], 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "hostnames": [], 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "hostnames_s3website": [], 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "master_zone": "9e6ed268-ecdc-4e24-ae7d-711f622a25df", 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "zones": [ 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "id": "9e6ed268-ecdc-4e24-ae7d-711f622a25df", 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "name": "z", 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "endpoints": [], 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "log_meta": "false", 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "log_data": "false", 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "bucket_index_max_shards": 11, 2024-05-20T22:33:02.353 INFO:teuthology.orchestra.run.smithi055.stdout: "read_only": "false", 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: "tier_type": "", 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: "sync_from_all": "true", 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: "sync_from": [], 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: "redirect_zone": "" 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: "placement_targets": [ 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: "name": "default-placement", 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: "tags": [], 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: "storage_classes": [ 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: "STANDARD" 2024-05-20T22:33:02.354 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: "default_placement": "default-placement", 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: "realm_id": "b8afa262-078e-4a92-9a54-059cd22c53f1", 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: "sync_policy": { 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: "groups": [] 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: "short_zone_ids": [ 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: "key": "9e6ed268-ecdc-4e24-ae7d-711f622a25df", 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: "val": 2556112588 2024-05-20T22:33:02.355 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "master_zonegroup": "e72c7d0d-20be-4681-b89e-6add7b1be9d6", 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "master_zone": "9e6ed268-ecdc-4e24-ae7d-711f622a25df", 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "period_config": { 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "bucket_quota": { 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "enabled": false, 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "check_on_raw": false, 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "max_size": -1, 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "max_size_kb": 0, 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "max_objects": -1 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:33:02.356 INFO:teuthology.orchestra.run.smithi055.stdout: "user_quota": { 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "enabled": false, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "check_on_raw": false, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "max_size": -1, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "max_size_kb": 0, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "max_objects": -1 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "user_ratelimit": { 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "max_read_ops": 0, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "max_write_ops": 0, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "max_read_bytes": 0, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "max_write_bytes": 0, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "enabled": false 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:33:02.357 INFO:teuthology.orchestra.run.smithi055.stdout: "bucket_ratelimit": { 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "max_read_ops": 0, 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "max_write_ops": 0, 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "max_read_bytes": 0, 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "max_write_bytes": 0, 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "enabled": false 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "anonymous_ratelimit": { 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "max_read_ops": 0, 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "max_write_ops": 0, 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "max_read_bytes": 0, 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "max_write_bytes": 0, 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: "enabled": false 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:33:02.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:33:02.359 INFO:teuthology.orchestra.run.smithi055.stdout: "realm_id": "b8afa262-078e-4a92-9a54-059cd22c53f1", 2024-05-20T22:33:02.359 INFO:teuthology.orchestra.run.smithi055.stdout: "realm_name": "r", 2024-05-20T22:33:02.359 INFO:teuthology.orchestra.run.smithi055.stdout: "realm_epoch": 2 2024-05-20T22:33:02.359 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:33:03.015 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-05-20T22:33:03.189 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 podman[49338]: 2024-05-20 22:33:03.148120317 +0000 UTC m=+1.194216829 container init 37600d6e6b2f10d71c806a5f11df501466ff201f46d0eccdfb577a76fc5ca17d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:03.189 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 podman[49338]: 2024-05-20 22:33:03.151701011 +0000 UTC m=+1.197797525 container start 37600d6e6b2f10d71c806a5f11df501466ff201f46d0eccdfb577a76fc5ca17d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:03.189 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:03.163Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-20T22:33:03.189 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:03.163Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-20T22:33:03.189 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:03.164Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.55 port=9094 2024-05-20T22:33:03.189 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:03.165Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-20T22:33:03.189 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:03.191Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-20T22:33:03.189 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:03.191Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-20T22:33:03.189 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:03.194Z caller=main.go:518 msg=Listening address=:9093 2024-05-20T22:33:03.190 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:03.194Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-20T22:33:03.486 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 bash[49338]: 37600d6e6b2f10d71c806a5f11df501466ff201f46d0eccdfb577a76fc5ca17d 2024-05-20T22:33:03.486 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:03 smithi055 systemd[1]: Started Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:33:03.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-20T22:33:03.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-mon[24757]: osdmap e71: 8 total, 8 up, 8 in 2024-05-20T22:33:03.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-mon[24757]: pgmap v62: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-05-20T22:33:03.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:03 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:03.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:03 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-20T22:33:03.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:03 smithi055 ceph-mon[29498]: osdmap e71: 8 total, 8 up, 8 in 2024-05-20T22:33:03.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:03 smithi055 ceph-mon[29498]: pgmap v62: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-05-20T22:33:03.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:03 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:03.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:03 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1948985022' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-20T22:33:03.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:03 smithi138 ceph-mon[25673]: osdmap e71: 8 total, 8 up, 8 in 2024-05-20T22:33:03.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:03 smithi138 ceph-mon[25673]: pgmap v62: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-05-20T22:33:03.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:03 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:04.568 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:04 smithi055 ceph-mon[24757]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:33:04.569 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:04 smithi055 ceph-mon[24757]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:33:04.569 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:04 smithi055 ceph-mon[29498]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:33:04.569 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:04 smithi055 ceph-mon[29498]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:33:04.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:04 smithi138 ceph-mon[25673]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:33:04.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:04 smithi138 ceph-mon[25673]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:33:05.087 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled rgw.foo update... 2024-05-20T22:33:05.392 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:05 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:05.166Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000546181s 2024-05-20T22:33:05.719 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph osd pool create foo' 2024-05-20T22:33:06.000 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:33:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:33:06.000 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:05 smithi138 systemd[1]: Stopping Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:33:06.001 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:05 smithi138 bash[42338]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:33:06.251 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:06 smithi138 ceph-mon[25673]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-05-20T22:33:06.251 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:06 smithi138 ceph-mon[25673]: from='client.24577 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:06.252 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:06 smithi138 ceph-mon[25673]: Saving service rgw.foo spec with placement count:2 2024-05-20T22:33:06.252 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:06 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:06.252 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.005Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-20T22:33:06.252 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-20T22:33:06.252 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-20T22:33:06.252 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-20T22:33:06.252 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-20T22:33:06.252 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-20T22:33:06.252 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-20T22:33:06.252 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-20T22:33:06.252 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-20T22:33:06.253 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-20T22:33:06.253 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-20T22:33:06.253 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[40721]: ts=2024-05-20T22:33:06.006Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-20T22:33:06.253 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 podman[42350]: 2024-05-20 22:33:06.016199254 +0000 UTC m=+0.120525473 container died 18fd7fbd48b74b1f552c24052cb4a5abd77e1d8287f43eda609422de968f3b15 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:06.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:06 smithi055 ceph-mon[29498]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-05-20T22:33:06.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:06 smithi055 ceph-mon[29498]: from='client.24577 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:06.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:06 smithi055 ceph-mon[29498]: Saving service rgw.foo spec with placement count:2 2024-05-20T22:33:06.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:06 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:06.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:06 smithi055 ceph-mon[24757]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-05-20T22:33:06.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:06 smithi055 ceph-mon[24757]: from='client.24577 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:06.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:06 smithi055 ceph-mon[24757]: Saving service rgw.foo spec with placement count:2 2024-05-20T22:33:06.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:06 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:06.519 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 podman[42350]: 2024-05-20 22:33:06.348439785 +0000 UTC m=+0.452766026 container cleanup 18fd7fbd48b74b1f552c24052cb4a5abd77e1d8287f43eda609422de968f3b15 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:06.519 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 bash[42350]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a 2024-05-20T22:33:06.777 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 podman[42361]: 2024-05-20 22:33:06.52471641 +0000 UTC m=+0.510199791 container remove 18fd7fbd48b74b1f552c24052cb4a5abd77e1d8287f43eda609422de968f3b15 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:06.778 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 bash[42374]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:33:06.778 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-20T22:33:06.778 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 systemd[1]: Stopped Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:33:06.778 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 systemd[1]: Starting Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:33:07.049 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 podman[42438]: 2024-05-20 22:33:06.686889312 +0000 UTC m=+0.017706718 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-20T22:33:07.049 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:06 smithi138 podman[42438]: 2024-05-20 22:33:06.811220263 +0000 UTC m=+0.142037707 container create 1c5c1e535c2dcc4a869b6ff30c202bc785a9c5d3b8bf641c8af36a1bc925d43d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:07.049 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 podman[42438]: 2024-05-20 22:33:07.027879966 +0000 UTC m=+0.358697368 container init 1c5c1e535c2dcc4a869b6ff30c202bc785a9c5d3b8bf641c8af36a1bc925d43d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:07.049 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 podman[42438]: 2024-05-20 22:33:07.030468878 +0000 UTC m=+0.361286290 container start 1c5c1e535c2dcc4a869b6ff30c202bc785a9c5d3b8bf641c8af36a1bc925d43d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:33:07.359 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.054Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-20T22:33:07.359 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.054Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-20T22:33:07.359 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.054Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-20T22:33:07.359 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.054Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi138 (none))" 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.054Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.054Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.056Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.057Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.058Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.060Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.060Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.231µs 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.060Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 bash[42438]: 1c5c1e535c2dcc4a869b6ff30c202bc785a9c5d3b8bf641c8af36a1bc925d43d 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 systemd[1]: Started Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.276Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.276Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-05-20T22:33:07.360 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.276Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=26.118µs wal_replay_duration=216.525059ms total_replay_duration=216.569082ms 2024-05-20T22:33:07.361 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.278Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-20T22:33:07.361 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.278Z caller=main.go:947 level=info msg="TSDB started" 2024-05-20T22:33:07.361 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.278Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-20T22:33:07.361 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.294Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.057426ms db_storage=895ns remote_storage=2.047µs web_handler=518ns query_engine=884ns scrape=554.779µs scrape_sd=43.929µs notify=38.521µs notify_sd=18.039µs rules=14.3873ms 2024-05-20T22:33:07.361 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:33:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:33:07.294Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-20T22:33:08.197 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'foo' created 2024-05-20T22:33:08.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1012 B/s wr, 2 op/s 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.55:9093"}]: dispatch 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.55:9093"}]: dispatch 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.138:9095"}]: dispatch 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.138:9095"}]: dispatch 2024-05-20T22:33:08.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.138:3000"}]: dispatch 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.138:3000"}]: dispatch 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1453272615' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1012 B/s wr, 2 op/s 2024-05-20T22:33:08.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.55:9093"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.55:9093"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.138:9095"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.138:9095"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.138:3000"}]: dispatch 2024-05-20T22:33:08.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.138:3000"}]: dispatch 2024-05-20T22:33:08.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:08.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:08.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:08.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1453272615' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-20T22:33:08.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:08 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-20T22:33:08.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: pgmap v64: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1012 B/s wr, 2 op/s 2024-05-20T22:33:08.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:33:08.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:33:08.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.55:9093"}]: dispatch 2024-05-20T22:33:08.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.55:9093"}]: dispatch 2024-05-20T22:33:08.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.138:9095"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.138:9095"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.138:3000"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.138:3000"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:08.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:08.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1453272615' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-20T22:33:08.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:08 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-20T22:33:08.903 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'rbd pool init foo' 2024-05-20T22:33:09.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:09 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-20T22:33:09.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:09 smithi055 ceph-mon[24757]: osdmap e72: 8 total, 8 up, 8 in 2024-05-20T22:33:09.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:09 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-20T22:33:09.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:09 smithi055 ceph-mon[29498]: osdmap e72: 8 total, 8 up, 8 in 2024-05-20T22:33:09.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:09 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-20T22:33:09.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:09 smithi138 ceph-mon[25673]: osdmap e72: 8 total, 8 up, 8 in 2024-05-20T22:33:10.468 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:10 smithi055 ceph-mon[24757]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-05-20T22:33:10.468 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:10 smithi055 ceph-mon[24757]: osdmap e73: 8 total, 8 up, 8 in 2024-05-20T22:33:10.468 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:10 smithi055 ceph-mon[29498]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-05-20T22:33:10.468 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:10 smithi055 ceph-mon[29498]: osdmap e73: 8 total, 8 up, 8 in 2024-05-20T22:33:10.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:10 smithi138 ceph-mon[25673]: pgmap v66: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-05-20T22:33:10.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:10 smithi138 ceph-mon[25673]: osdmap e73: 8 total, 8 up, 8 in 2024-05-20T22:33:11.449 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:11 smithi055 ceph-mon[24757]: osdmap e74: 8 total, 8 up, 8 in 2024-05-20T22:33:11.449 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:11 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:11.449 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:11 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/856893803' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-20T22:33:11.449 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:11 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-20T22:33:11.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:11 smithi055 ceph-mon[29498]: osdmap e74: 8 total, 8 up, 8 in 2024-05-20T22:33:11.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:11 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:11.450 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:11 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/856893803' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-20T22:33:11.450 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:11 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-20T22:33:11.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:11 smithi138 ceph-mon[25673]: osdmap e74: 8 total, 8 up, 8 in 2024-05-20T22:33:11.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:11 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:11.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:11 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/856893803' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-20T22:33:11.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:11 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-20T22:33:12.447 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:12 smithi055 ceph-mon[29498]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 0 op/s 2024-05-20T22:33:12.447 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:12 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-20T22:33:12.447 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:12 smithi055 ceph-mon[29498]: osdmap e75: 8 total, 8 up, 8 in 2024-05-20T22:33:12.448 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:12 smithi055 ceph-mon[24757]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 0 op/s 2024-05-20T22:33:12.448 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:12 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-20T22:33:12.448 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:12 smithi055 ceph-mon[24757]: osdmap e75: 8 total, 8 up, 8 in 2024-05-20T22:33:12.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:12 smithi138 ceph-mon[25673]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 0 op/s 2024-05-20T22:33:12.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:12 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-20T22:33:12.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:12 smithi138 ceph-mon[25673]: osdmap e75: 8 total, 8 up, 8 in 2024-05-20T22:33:13.423 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:33:13.168Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002806753s 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: osdmap e76: 8 total, 8 up, 8 in 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: Saving service rgw.foo spec with placement count:2 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:13.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[24757]: Deploying daemon rgw.foo.smithi055.ymnqng on smithi055 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: osdmap e76: 8 total, 8 up, 8 in 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: Saving service rgw.foo spec with placement count:2 2024-05-20T22:33:13.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:13.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:13.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:33:13.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:13.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:13 smithi055 ceph-mon[29498]: Deploying daemon rgw.foo.smithi055.ymnqng on smithi055 2024-05-20T22:33:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: osdmap e76: 8 total, 8 up, 8 in 2024-05-20T22:33:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: Saving service rgw.foo spec with placement count:2 2024-05-20T22:33:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:33:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:13.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:13 smithi138 ceph-mon[25673]: Deploying daemon rgw.foo.smithi055.ymnqng on smithi055 2024-05-20T22:33:14.039 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch apply iscsi foo u p' 2024-05-20T22:33:14.459 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:14 smithi055 ceph-mon[29498]: osdmap e77: 8 total, 8 up, 8 in 2024-05-20T22:33:14.459 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:14 smithi055 ceph-mon[24757]: osdmap e77: 8 total, 8 up, 8 in 2024-05-20T22:33:14.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:14 smithi138 ceph-mon[25673]: osdmap e77: 8 total, 8 up, 8 in 2024-05-20T22:33:15.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:15 smithi055 ceph-mon[24757]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:15.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:15 smithi055 ceph-mon[29498]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:15.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:15 smithi138 ceph-mon[25673]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:16.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:33:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:33:16.716 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled iscsi.foo update... 2024-05-20T22:33:17.421 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:33:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:17] "GET /metrics HTTP/1.1" 200 197860 "" "Prometheus/2.33.4" 2024-05-20T22:33:17.447 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'sleep 180' 2024-05-20T22:33:17.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:17 smithi055 ceph-mon[24757]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:17.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:17 smithi055 ceph-mon[24757]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:17.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:17 smithi055 ceph-mon[24757]: Saving service iscsi.foo spec with placement count:1 2024-05-20T22:33:17.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:17 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:17.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:17 smithi055 ceph-mon[29498]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:17.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:17 smithi055 ceph-mon[29498]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:17.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:17 smithi055 ceph-mon[29498]: Saving service iscsi.foo spec with placement count:1 2024-05-20T22:33:17.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:17 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:18.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:17 smithi138 ceph-mon[25673]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:18.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:17 smithi138 ceph-mon[25673]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:18.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:17 smithi138 ceph-mon[25673]: Saving service iscsi.foo spec with placement count:1 2024-05-20T22:33:18.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:17 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:19.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:19.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:19.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:19.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:33:19.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:19.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:19.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[24757]: Deploying daemon rgw.foo.smithi138.sfpuja on smithi138 2024-05-20T22:33:19.019 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:19.019 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:19.019 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:19.019 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:33:19.019 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:19.019 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:19.019 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:18 smithi055 ceph-mon[29498]: Deploying daemon rgw.foo.smithi138.sfpuja on smithi138 2024-05-20T22:33:19.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:18 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:19.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:18 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:19.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:18 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:19.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:18 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:33:19.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:18 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:19.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:18 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:19.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:18 smithi138 ceph-mon[25673]: Deploying daemon rgw.foo.smithi138.sfpuja on smithi138 2024-05-20T22:33:20.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:19 smithi138 ceph-mon[25673]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 292 B/s wr, 0 op/s 2024-05-20T22:33:20.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:19 smithi055 ceph-mon[24757]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 292 B/s wr, 0 op/s 2024-05-20T22:33:20.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:19 smithi055 ceph-mon[29498]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 292 B/s wr, 0 op/s 2024-05-20T22:33:22.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:21 smithi138 ceph-mon[25673]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-05-20T22:33:22.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:21 smithi055 ceph-mon[24757]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-05-20T22:33:22.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:21 smithi055 ceph-mon[29498]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-05-20T22:33:23.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:23 smithi055 ceph-mon[24757]: osdmap e78: 8 total, 8 up, 8 in 2024-05-20T22:33:23.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:23 smithi055 ceph-mon[29498]: osdmap e78: 8 total, 8 up, 8 in 2024-05-20T22:33:23.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:23 smithi138 ceph-mon[25673]: osdmap e78: 8 total, 8 up, 8 in 2024-05-20T22:33:24.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:24 smithi138 ceph-mon[25673]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 3.9 KiB/s wr, 124 op/s 2024-05-20T22:33:24.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:24 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:24.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:24 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:24.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:24 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:24.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:24 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[24757]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 3.9 KiB/s wr, 124 op/s 2024-05-20T22:33:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:24.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[29498]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 3.9 KiB/s wr, 124 op/s 2024-05-20T22:33:24.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:24.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:24.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:24.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:24 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:25.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:33:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:33:26.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:26 smithi055 ceph-mon[24757]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-05-20T22:33:26.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:26 smithi055 ceph-mon[29498]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-05-20T22:33:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:26 smithi138 ceph-mon[25673]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-05-20T22:33:27.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:33:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:27] "GET /metrics HTTP/1.1" 200 207239 "" "Prometheus/2.33.4" 2024-05-20T22:33:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-05-20T22:33:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:33:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:33:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:33:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:33:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:33:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-05-20T22:33:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-05-20T22:33:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-05-20T22:33:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-05-20T22:33:27.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-05-20T22:33:27.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:33:27.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:33:27.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:33:27.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:33:27.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:27.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:28.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:28 smithi138 ceph-mon[25673]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-05-20T22:33:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:28 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-05-20T22:33:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:28 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-05-20T22:33:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:28 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-05-20T22:33:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:28 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-05-20T22:33:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:28 smithi138 ceph-mon[25673]: osdmap e79: 8 total, 8 up, 8 in 2024-05-20T22:33:28.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:28 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:28.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[24757]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-05-20T22:33:28.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-05-20T22:33:28.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-05-20T22:33:28.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-05-20T22:33:28.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-05-20T22:33:28.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[24757]: osdmap e79: 8 total, 8 up, 8 in 2024-05-20T22:33:28.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:28.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[29498]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-05-20T22:33:28.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-05-20T22:33:28.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-05-20T22:33:28.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-05-20T22:33:28.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-05-20T22:33:28.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[29498]: osdmap e79: 8 total, 8 up, 8 in 2024-05-20T22:33:28.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:28 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[24757]: Checking dashboard <-> RGW credentials 2024-05-20T22:33:29.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[24757]: osdmap e80: 8 total, 8 up, 8 in 2024-05-20T22:33:29.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:33:29.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:33:29.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[29498]: Checking dashboard <-> RGW credentials 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[29498]: osdmap e80: 8 total, 8 up, 8 in 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-20T22:33:29.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:29 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:29.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:29 smithi138 ceph-mon[25673]: Checking dashboard <-> RGW credentials 2024-05-20T22:33:29.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:29 smithi138 ceph-mon[25673]: osdmap e80: 8 total, 8 up, 8 in 2024-05-20T22:33:29.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:29.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:29 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:33:29.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:33:29.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:29 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-20T22:33:29.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:29 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:30.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:30 smithi055 ceph-mon[24757]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 180 KiB/s rd, 5.5 KiB/s wr, 344 op/s 2024-05-20T22:33:30.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:30 smithi055 ceph-mon[24757]: Checking pool "foo" exists for service iscsi.foo 2024-05-20T22:33:30.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:30 smithi055 ceph-mon[24757]: Deploying daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:33:30.351 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:30 smithi055 ceph-mon[29498]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 180 KiB/s rd, 5.5 KiB/s wr, 344 op/s 2024-05-20T22:33:30.351 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:30 smithi055 ceph-mon[29498]: Checking pool "foo" exists for service iscsi.foo 2024-05-20T22:33:30.351 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:30 smithi055 ceph-mon[29498]: Deploying daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:33:30.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:30 smithi138 ceph-mon[25673]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 180 KiB/s rd, 5.5 KiB/s wr, 344 op/s 2024-05-20T22:33:30.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:30 smithi138 ceph-mon[25673]: Checking pool "foo" exists for service iscsi.foo 2024-05-20T22:33:30.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:30 smithi138 ceph-mon[25673]: Deploying daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:33:31.437 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:31 smithi055 ceph-mon[24757]: mgrmap e21: y(active, since 2m), standbys: x 2024-05-20T22:33:31.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:31 smithi055 ceph-mon[29498]: mgrmap e21: y(active, since 2m), standbys: x 2024-05-20T22:33:31.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:31 smithi138 ceph-mon[25673]: mgrmap e21: y(active, since 2m), standbys: x 2024-05-20T22:33:32.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[24757]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 127 B/s wr, 125 op/s 2024-05-20T22:33:32.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:32.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3805618492' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[29498]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 127 B/s wr, 125 op/s 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:32.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:32 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3805618492' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:33:32.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:32 smithi138 ceph-mon[25673]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 127 B/s wr, 125 op/s 2024-05-20T22:33:32.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:32 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:32.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:32 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:32.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:32 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:32.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:32 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:32.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:32 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:32.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:32 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3805618492' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:33:33.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:33 smithi055 ceph-mon[24757]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 511 B/s wr, 131 op/s 2024-05-20T22:33:33.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:33 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1934927179' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2680501974"}]: dispatch 2024-05-20T22:33:33.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:33 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2680501974"}]: dispatch 2024-05-20T22:33:33.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:33 smithi055 ceph-mon[29498]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 511 B/s wr, 131 op/s 2024-05-20T22:33:33.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:33 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1934927179' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2680501974"}]: dispatch 2024-05-20T22:33:33.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:33 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2680501974"}]: dispatch 2024-05-20T22:33:33.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:33 smithi138 ceph-mon[25673]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 511 B/s wr, 131 op/s 2024-05-20T22:33:33.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:33 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1934927179' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2680501974"}]: dispatch 2024-05-20T22:33:33.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:33 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2680501974"}]: dispatch 2024-05-20T22:33:34.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:34 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2680501974"}]': finished 2024-05-20T22:33:34.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:34 smithi055 ceph-mon[24757]: osdmap e81: 8 total, 8 up, 8 in 2024-05-20T22:33:34.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:34 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3701530687' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2905501019"}]: dispatch 2024-05-20T22:33:34.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:34 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2680501974"}]': finished 2024-05-20T22:33:34.350 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:34 smithi055 ceph-mon[29498]: osdmap e81: 8 total, 8 up, 8 in 2024-05-20T22:33:34.350 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:34 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3701530687' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2905501019"}]: dispatch 2024-05-20T22:33:34.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:34 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2680501974"}]': finished 2024-05-20T22:33:34.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:34 smithi138 ceph-mon[25673]: osdmap e81: 8 total, 8 up, 8 in 2024-05-20T22:33:34.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:34 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3701530687' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2905501019"}]: dispatch 2024-05-20T22:33:35.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3701530687' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2905501019"}]': finished 2024-05-20T22:33:35.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[29498]: osdmap e82: 8 total, 8 up, 8 in 2024-05-20T22:33:35.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[29498]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 501 B/s wr, 8 op/s 2024-05-20T22:33:35.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/851154614' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/429623249"}]: dispatch 2024-05-20T22:33:35.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/429623249"}]: dispatch 2024-05-20T22:33:35.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:35.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:35.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3701530687' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2905501019"}]': finished 2024-05-20T22:33:35.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[24757]: osdmap e82: 8 total, 8 up, 8 in 2024-05-20T22:33:35.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[24757]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 501 B/s wr, 8 op/s 2024-05-20T22:33:35.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/851154614' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/429623249"}]: dispatch 2024-05-20T22:33:35.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/429623249"}]: dispatch 2024-05-20T22:33:35.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:35.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:35 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:35 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3701530687' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2905501019"}]': finished 2024-05-20T22:33:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:35 smithi138 ceph-mon[25673]: osdmap e82: 8 total, 8 up, 8 in 2024-05-20T22:33:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:35 smithi138 ceph-mon[25673]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 501 B/s wr, 8 op/s 2024-05-20T22:33:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:35 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/851154614' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/429623249"}]: dispatch 2024-05-20T22:33:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:35 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/429623249"}]: dispatch 2024-05-20T22:33:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:35 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:35.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:35 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:33:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:33:36.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/429623249"}]': finished 2024-05-20T22:33:36.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: osdmap e83: 8 total, 8 up, 8 in 2024-05-20T22:33:36.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:33:36.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:33:36.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: Adding iSCSI gateway http://:@172.21.15.55:5000 to Dashboard 2024-05-20T22:33:36.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:33:36.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi055"}]: dispatch 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi055"}]: dispatch 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: Checking dashboard <-> RGW credentials 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2851663445' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3886325575"}]: dispatch 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3886325575"}]: dispatch 2024-05-20T22:33:36.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/429623249"}]': finished 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: osdmap e83: 8 total, 8 up, 8 in 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: Adding iSCSI gateway http://:@172.21.15.55:5000 to Dashboard 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi055"}]: dispatch 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi055"}]: dispatch 2024-05-20T22:33:36.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:36.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:36.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:36.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: Checking dashboard <-> RGW credentials 2024-05-20T22:33:36.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2851663445' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3886325575"}]: dispatch 2024-05-20T22:33:36.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3886325575"}]: dispatch 2024-05-20T22:33:36.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:36 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/429623249"}]': finished 2024-05-20T22:33:36.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: osdmap e83: 8 total, 8 up, 8 in 2024-05-20T22:33:36.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:33:36.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:33:36.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: Adding iSCSI gateway http://:@172.21.15.55:5000 to Dashboard 2024-05-20T22:33:36.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:33:36.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi055"}]: dispatch 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi055"}]: dispatch 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: Checking dashboard <-> RGW credentials 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2851663445' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3886325575"}]: dispatch 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3886325575"}]: dispatch 2024-05-20T22:33:36.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:36 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:33:37.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:33:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:37] "GET /metrics HTTP/1.1" 200 207239 "" "Prometheus/2.33.4" 2024-05-20T22:33:37.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:37 smithi138 ceph-mon[25673]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 511 B/s wr, 9 op/s 2024-05-20T22:33:37.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:37 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3886325575"}]': finished 2024-05-20T22:33:37.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:37 smithi138 ceph-mon[25673]: osdmap e84: 8 total, 8 up, 8 in 2024-05-20T22:33:37.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:37 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1498108456' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/388134085"}]: dispatch 2024-05-20T22:33:37.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:37 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/388134085"}]: dispatch 2024-05-20T22:33:37.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[24757]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 511 B/s wr, 9 op/s 2024-05-20T22:33:37.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3886325575"}]': finished 2024-05-20T22:33:37.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[24757]: osdmap e84: 8 total, 8 up, 8 in 2024-05-20T22:33:37.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1498108456' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/388134085"}]: dispatch 2024-05-20T22:33:37.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/388134085"}]: dispatch 2024-05-20T22:33:37.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[29498]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 511 B/s wr, 9 op/s 2024-05-20T22:33:37.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3886325575"}]': finished 2024-05-20T22:33:37.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[29498]: osdmap e84: 8 total, 8 up, 8 in 2024-05-20T22:33:37.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1498108456' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/388134085"}]: dispatch 2024-05-20T22:33:37.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:37 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/388134085"}]: dispatch 2024-05-20T22:33:39.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:38 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/388134085"}]': finished 2024-05-20T22:33:39.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:38 smithi138 ceph-mon[25673]: osdmap e85: 8 total, 8 up, 8 in 2024-05-20T22:33:39.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:38 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2113979276' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1787159779"}]: dispatch 2024-05-20T22:33:39.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:38 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1787159779"}]: dispatch 2024-05-20T22:33:39.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:38 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/388134085"}]': finished 2024-05-20T22:33:39.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:38 smithi055 ceph-mon[24757]: osdmap e85: 8 total, 8 up, 8 in 2024-05-20T22:33:39.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:38 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2113979276' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1787159779"}]: dispatch 2024-05-20T22:33:39.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:38 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1787159779"}]: dispatch 2024-05-20T22:33:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:38 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/388134085"}]': finished 2024-05-20T22:33:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:38 smithi055 ceph-mon[29498]: osdmap e85: 8 total, 8 up, 8 in 2024-05-20T22:33:39.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:38 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2113979276' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1787159779"}]: dispatch 2024-05-20T22:33:39.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:38 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1787159779"}]: dispatch 2024-05-20T22:33:40.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:39 smithi138 ceph-mon[25673]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-05-20T22:33:40.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:39 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1787159779"}]': finished 2024-05-20T22:33:40.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:39 smithi138 ceph-mon[25673]: osdmap e86: 8 total, 8 up, 8 in 2024-05-20T22:33:40.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:39 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3050310856' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1640123401"}]: dispatch 2024-05-20T22:33:40.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:39 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1640123401"}]: dispatch 2024-05-20T22:33:40.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[24757]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-05-20T22:33:40.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1787159779"}]': finished 2024-05-20T22:33:40.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[24757]: osdmap e86: 8 total, 8 up, 8 in 2024-05-20T22:33:40.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3050310856' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1640123401"}]: dispatch 2024-05-20T22:33:40.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1640123401"}]: dispatch 2024-05-20T22:33:40.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[29498]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-05-20T22:33:40.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1787159779"}]': finished 2024-05-20T22:33:40.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[29498]: osdmap e86: 8 total, 8 up, 8 in 2024-05-20T22:33:40.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3050310856' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1640123401"}]: dispatch 2024-05-20T22:33:40.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:39 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1640123401"}]: dispatch 2024-05-20T22:33:41.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:40 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1640123401"}]': finished 2024-05-20T22:33:41.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:40 smithi055 ceph-mon[24757]: osdmap e87: 8 total, 8 up, 8 in 2024-05-20T22:33:41.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:40 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3342628090' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3211423299"}]: dispatch 2024-05-20T22:33:41.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:40 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3211423299"}]: dispatch 2024-05-20T22:33:41.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:40 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1640123401"}]': finished 2024-05-20T22:33:41.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:40 smithi055 ceph-mon[29498]: osdmap e87: 8 total, 8 up, 8 in 2024-05-20T22:33:41.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:40 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3342628090' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3211423299"}]: dispatch 2024-05-20T22:33:41.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:40 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3211423299"}]: dispatch 2024-05-20T22:33:41.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:40 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1640123401"}]': finished 2024-05-20T22:33:41.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:40 smithi138 ceph-mon[25673]: osdmap e87: 8 total, 8 up, 8 in 2024-05-20T22:33:41.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:40 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3342628090' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3211423299"}]: dispatch 2024-05-20T22:33:41.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:40 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3211423299"}]: dispatch 2024-05-20T22:33:42.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[24757]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-05-20T22:33:42.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3211423299"}]': finished 2024-05-20T22:33:42.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[24757]: osdmap e88: 8 total, 8 up, 8 in 2024-05-20T22:33:42.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1829476647' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/388134085"}]: dispatch 2024-05-20T22:33:42.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/388134085"}]: dispatch 2024-05-20T22:33:42.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[29498]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-05-20T22:33:42.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3211423299"}]': finished 2024-05-20T22:33:42.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[29498]: osdmap e88: 8 total, 8 up, 8 in 2024-05-20T22:33:42.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1829476647' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/388134085"}]: dispatch 2024-05-20T22:33:42.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:41 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/388134085"}]: dispatch 2024-05-20T22:33:42.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:41 smithi138 ceph-mon[25673]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-05-20T22:33:42.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:41 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3211423299"}]': finished 2024-05-20T22:33:42.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:41 smithi138 ceph-mon[25673]: osdmap e88: 8 total, 8 up, 8 in 2024-05-20T22:33:42.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:41 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1829476647' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/388134085"}]: dispatch 2024-05-20T22:33:42.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:41 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/388134085"}]: dispatch 2024-05-20T22:33:43.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:42 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:33:43.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:42 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/388134085"}]': finished 2024-05-20T22:33:43.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:42 smithi055 ceph-mon[24757]: osdmap e89: 8 total, 8 up, 8 in 2024-05-20T22:33:43.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:42 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2370732537' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2635876549"}]: dispatch 2024-05-20T22:33:43.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:42 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:33:43.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:42 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/388134085"}]': finished 2024-05-20T22:33:43.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:42 smithi055 ceph-mon[29498]: osdmap e89: 8 total, 8 up, 8 in 2024-05-20T22:33:43.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:42 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2370732537' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2635876549"}]: dispatch 2024-05-20T22:33:43.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:42 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:33:43.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:42 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/388134085"}]': finished 2024-05-20T22:33:43.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:42 smithi138 ceph-mon[25673]: osdmap e89: 8 total, 8 up, 8 in 2024-05-20T22:33:43.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:42 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2370732537' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2635876549"}]: dispatch 2024-05-20T22:33:44.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:44 smithi138 ceph-mon[25673]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:44.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:44 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2370732537' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2635876549"}]': finished 2024-05-20T22:33:44.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:44 smithi138 ceph-mon[25673]: osdmap e90: 8 total, 8 up, 8 in 2024-05-20T22:33:44.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:44 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2964806445' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3728861281"}]: dispatch 2024-05-20T22:33:44.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:44 smithi055 ceph-mon[24757]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:44.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:44 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2370732537' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2635876549"}]': finished 2024-05-20T22:33:44.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:44 smithi055 ceph-mon[24757]: osdmap e90: 8 total, 8 up, 8 in 2024-05-20T22:33:44.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:44 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2964806445' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3728861281"}]: dispatch 2024-05-20T22:33:44.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:44 smithi055 ceph-mon[29498]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:44.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:44 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2370732537' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2635876549"}]': finished 2024-05-20T22:33:44.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:44 smithi055 ceph-mon[29498]: osdmap e90: 8 total, 8 up, 8 in 2024-05-20T22:33:44.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:44 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2964806445' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3728861281"}]: dispatch 2024-05-20T22:33:45.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:45 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2964806445' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3728861281"}]': finished 2024-05-20T22:33:45.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:45 smithi138 ceph-mon[25673]: osdmap e91: 8 total, 8 up, 8 in 2024-05-20T22:33:45.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:45 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/670911633' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3366104370"}]: dispatch 2024-05-20T22:33:45.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:45 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2964806445' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3728861281"}]': finished 2024-05-20T22:33:45.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:45 smithi055 ceph-mon[24757]: osdmap e91: 8 total, 8 up, 8 in 2024-05-20T22:33:45.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:45 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/670911633' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3366104370"}]: dispatch 2024-05-20T22:33:45.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:45 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2964806445' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3728861281"}]': finished 2024-05-20T22:33:45.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:45 smithi055 ceph-mon[29498]: osdmap e91: 8 total, 8 up, 8 in 2024-05-20T22:33:45.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:45 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/670911633' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3366104370"}]: dispatch 2024-05-20T22:33:46.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:33:45 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:33:46.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:46 smithi138 ceph-mon[25673]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:46.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:46 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/670911633' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3366104370"}]': finished 2024-05-20T22:33:46.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:46 smithi138 ceph-mon[25673]: osdmap e92: 8 total, 8 up, 8 in 2024-05-20T22:33:46.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:46 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/4024158858' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/429623249"}]: dispatch 2024-05-20T22:33:46.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:46 smithi055 ceph-mon[24757]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:46.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:46 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/670911633' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3366104370"}]': finished 2024-05-20T22:33:46.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:46 smithi055 ceph-mon[24757]: osdmap e92: 8 total, 8 up, 8 in 2024-05-20T22:33:46.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:46 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4024158858' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/429623249"}]: dispatch 2024-05-20T22:33:46.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:46 smithi055 ceph-mon[29498]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:46.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:46 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/670911633' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3366104370"}]': finished 2024-05-20T22:33:46.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:46 smithi055 ceph-mon[29498]: osdmap e92: 8 total, 8 up, 8 in 2024-05-20T22:33:46.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:46 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/4024158858' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/429623249"}]: dispatch 2024-05-20T22:33:47.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:33:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:47] "GET /metrics HTTP/1.1" 200 208094 "" "Prometheus/2.33.4" 2024-05-20T22:33:47.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:47 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/4024158858' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/429623249"}]': finished 2024-05-20T22:33:47.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:47 smithi055 ceph-mon[29498]: osdmap e93: 8 total, 8 up, 8 in 2024-05-20T22:33:47.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:47 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2435736847' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1882487112"}]: dispatch 2024-05-20T22:33:47.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:47 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1882487112"}]: dispatch 2024-05-20T22:33:47.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:47 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4024158858' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/429623249"}]': finished 2024-05-20T22:33:47.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:47 smithi055 ceph-mon[24757]: osdmap e93: 8 total, 8 up, 8 in 2024-05-20T22:33:47.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:47 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2435736847' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1882487112"}]: dispatch 2024-05-20T22:33:47.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:47 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1882487112"}]: dispatch 2024-05-20T22:33:47.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:47 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/4024158858' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/429623249"}]': finished 2024-05-20T22:33:47.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:47 smithi138 ceph-mon[25673]: osdmap e93: 8 total, 8 up, 8 in 2024-05-20T22:33:47.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:47 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2435736847' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1882487112"}]: dispatch 2024-05-20T22:33:47.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:47 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1882487112"}]: dispatch 2024-05-20T22:33:48.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:48 smithi138 ceph-mon[25673]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:48.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:48 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1882487112"}]': finished 2024-05-20T22:33:48.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:48 smithi138 ceph-mon[25673]: osdmap e94: 8 total, 8 up, 8 in 2024-05-20T22:33:48.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:48 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1593738761' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1950099066"}]: dispatch 2024-05-20T22:33:48.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:48 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1950099066"}]: dispatch 2024-05-20T22:33:48.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[24757]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:48.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1882487112"}]': finished 2024-05-20T22:33:48.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[24757]: osdmap e94: 8 total, 8 up, 8 in 2024-05-20T22:33:48.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1593738761' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1950099066"}]: dispatch 2024-05-20T22:33:48.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1950099066"}]: dispatch 2024-05-20T22:33:48.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[29498]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:33:48.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1882487112"}]': finished 2024-05-20T22:33:48.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[29498]: osdmap e94: 8 total, 8 up, 8 in 2024-05-20T22:33:48.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1593738761' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1950099066"}]: dispatch 2024-05-20T22:33:48.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:48 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1950099066"}]: dispatch 2024-05-20T22:33:49.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:49 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1950099066"}]': finished 2024-05-20T22:33:49.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:49 smithi138 ceph-mon[25673]: osdmap e95: 8 total, 8 up, 8 in 2024-05-20T22:33:49.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:49 smithi138 ceph-mon[25673]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:49.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:49 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/4182135161' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3728861281"}]: dispatch 2024-05-20T22:33:49.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:49 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1950099066"}]': finished 2024-05-20T22:33:49.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:49 smithi055 ceph-mon[29498]: osdmap e95: 8 total, 8 up, 8 in 2024-05-20T22:33:49.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:49 smithi055 ceph-mon[29498]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:49.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:49 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/4182135161' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3728861281"}]: dispatch 2024-05-20T22:33:49.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:49 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1950099066"}]': finished 2024-05-20T22:33:49.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:49 smithi055 ceph-mon[24757]: osdmap e95: 8 total, 8 up, 8 in 2024-05-20T22:33:49.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:49 smithi055 ceph-mon[24757]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:49.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:49 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4182135161' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3728861281"}]: dispatch 2024-05-20T22:33:50.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:50 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/4182135161' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3728861281"}]': finished 2024-05-20T22:33:50.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:50 smithi138 ceph-mon[25673]: osdmap e96: 8 total, 8 up, 8 in 2024-05-20T22:33:50.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:50 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/4182135161' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3728861281"}]': finished 2024-05-20T22:33:50.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:50 smithi055 ceph-mon[29498]: osdmap e96: 8 total, 8 up, 8 in 2024-05-20T22:33:50.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:50 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4182135161' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3728861281"}]': finished 2024-05-20T22:33:50.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:50 smithi055 ceph-mon[24757]: osdmap e96: 8 total, 8 up, 8 in 2024-05-20T22:33:51.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:51 smithi138 ceph-mon[25673]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:51.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:51 smithi055 ceph-mon[24757]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:51.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:51 smithi055 ceph-mon[29498]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:33:53.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:53 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:33:53.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:53 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:33:53.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:53 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:33:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:54 smithi055 ceph-mon[24757]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-20T22:33:54.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:54 smithi055 ceph-mon[29498]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-20T22:33:54.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:54 smithi138 ceph-mon[25673]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-20T22:33:56.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:33:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:33:56.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:56 smithi055 ceph-mon[24757]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-20T22:33:56.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:56 smithi055 ceph-mon[29498]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-20T22:33:56.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:56 smithi138 ceph-mon[25673]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-20T22:33:57.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:33:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:33:57] "GET /metrics HTTP/1.1" 200 208107 "" "Prometheus/2.33.4" 2024-05-20T22:33:58.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:33:58 smithi055 ceph-mon[24757]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-05-20T22:33:58.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:33:58 smithi055 ceph-mon[29498]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-05-20T22:33:58.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:33:58 smithi138 ceph-mon[25673]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-05-20T22:34:00.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:00 smithi055 ceph-mon[24757]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T22:34:00.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:00 smithi055 ceph-mon[29498]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T22:34:00.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:00 smithi138 ceph-mon[25673]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T22:34:02.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:02 smithi055 ceph-mon[24757]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-05-20T22:34:02.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:02 smithi055 ceph-mon[29498]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-05-20T22:34:02.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:02 smithi138 ceph-mon[25673]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-05-20T22:34:03.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:03 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:03.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:03 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:03.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:03 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:04.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:04 smithi138 ceph-mon[25673]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:04.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:04 smithi055 ceph-mon[24757]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:04.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:04 smithi055 ceph-mon[29498]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:06.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:34:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:34:06.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:06 smithi138 ceph-mon[25673]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:06.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:06 smithi055 ceph-mon[24757]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:06.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:06 smithi055 ceph-mon[29498]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:07.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:34:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:07] "GET /metrics HTTP/1.1" 200 208107 "" "Prometheus/2.33.4" 2024-05-20T22:34:08.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:08 smithi138 ceph-mon[25673]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:08.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:08 smithi055 ceph-mon[24757]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:08.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:08 smithi055 ceph-mon[29498]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:10.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:10 smithi138 ceph-mon[25673]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:10.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:10 smithi055 ceph-mon[24757]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:10.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:10 smithi055 ceph-mon[29498]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:11.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:11 smithi138 ceph-mon[25673]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:11.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:11 smithi055 ceph-mon[24757]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:11.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:11 smithi055 ceph-mon[29498]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:13.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:13 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:13.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:13 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:13.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:13 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:14.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:14 smithi055 ceph-mon[24757]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:14.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:14 smithi055 ceph-mon[29498]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:14.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:14 smithi138 ceph-mon[25673]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:16.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:34:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:34:16.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:16 smithi055 ceph-mon[24757]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:16.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:16 smithi055 ceph-mon[29498]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:16.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:16 smithi138 ceph-mon[25673]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:17.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:34:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:17] "GET /metrics HTTP/1.1" 200 208051 "" "Prometheus/2.33.4" 2024-05-20T22:34:18.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:18 smithi055 ceph-mon[24757]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:18.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:18 smithi055 ceph-mon[29498]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:18.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:18 smithi138 ceph-mon[25673]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:20.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:20 smithi055 ceph-mon[24757]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:20.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:20 smithi055 ceph-mon[29498]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:20.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:20 smithi138 ceph-mon[25673]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:22.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:22 smithi055 ceph-mon[24757]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:22.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:22 smithi055 ceph-mon[29498]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:22.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:22 smithi138 ceph-mon[25673]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:23 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:23.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:23 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:23.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:23 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:24.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:24 smithi138 ceph-mon[25673]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:24.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:24 smithi055 ceph-mon[24757]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:24.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:24 smithi055 ceph-mon[29498]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:34:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:34:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:26 smithi138 ceph-mon[25673]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:26.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:26 smithi055 ceph-mon[24757]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:26.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:26 smithi055 ceph-mon[29498]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:27.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:34:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:27] "GET /metrics HTTP/1.1" 200 208012 "" "Prometheus/2.33.4" 2024-05-20T22:34:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:34:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:34:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:34:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:34:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:34:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:34:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:34:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:34:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:34:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:34:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:34:27.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:34:28.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:28 smithi138 ceph-mon[25673]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:28.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:28 smithi055 ceph-mon[24757]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:28.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:28 smithi055 ceph-mon[29498]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:30.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:30 smithi138 ceph-mon[25673]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:30.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:30 smithi055 ceph-mon[24757]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:30.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:30 smithi055 ceph-mon[29498]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:32.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:32 smithi138 ceph-mon[25673]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:32.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:32 smithi055 ceph-mon[24757]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:32.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:32 smithi055 ceph-mon[29498]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:33.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:33 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:33.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:33 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:33.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:33 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:34.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:34 smithi138 ceph-mon[25673]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:34.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:34 smithi055 ceph-mon[24757]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:34.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:34 smithi055 ceph-mon[29498]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:35.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:35 smithi138 ceph-mon[25673]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:35.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:35 smithi055 ceph-mon[24757]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:35.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:35 smithi055 ceph-mon[29498]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:36.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:34:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:34:36.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:36.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:34:36.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:34:36.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:36 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:34:36.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:36.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:34:36.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:34:36.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:36 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:34:36.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:36.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:34:36.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:34:36.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:36 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:34:37.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:34:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:37] "GET /metrics HTTP/1.1" 200 208012 "" "Prometheus/2.33.4" 2024-05-20T22:34:38.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:38 smithi055 ceph-mon[29498]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:38.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:38 smithi055 ceph-mon[24757]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:38 smithi138 ceph-mon[25673]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:40.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:40 smithi055 ceph-mon[24757]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:40.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:40 smithi055 ceph-mon[29498]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:40.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:40 smithi138 ceph-mon[25673]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:42.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:42 smithi055 ceph-mon[24757]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:42.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:42 smithi055 ceph-mon[29498]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:42.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:42 smithi138 ceph-mon[25673]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:43.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:43 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:43 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:43.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:43 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:44.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:44 smithi138 ceph-mon[25673]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:44.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:44 smithi055 ceph-mon[24757]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:44.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:44 smithi055 ceph-mon[29498]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:46.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:34:45 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:34:46.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:46 smithi138 ceph-mon[25673]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:46.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:46 smithi055 ceph-mon[24757]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:46.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:46 smithi055 ceph-mon[29498]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:47.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:34:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:47] "GET /metrics HTTP/1.1" 200 208022 "" "Prometheus/2.33.4" 2024-05-20T22:34:48.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:48 smithi138 ceph-mon[25673]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:48.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:48 smithi055 ceph-mon[24757]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:48.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:48 smithi055 ceph-mon[29498]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:50.538 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:50 smithi055 ceph-mon[24757]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:50.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:50 smithi055 ceph-mon[29498]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:50.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:50 smithi138 ceph-mon[25673]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:52.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:52 smithi138 ceph-mon[25673]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:52.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:52 smithi055 ceph-mon[24757]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:52.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:52 smithi055 ceph-mon[29498]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:53.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:53 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:53.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:53 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:53.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:53 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:34:54.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:54 smithi138 ceph-mon[25673]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:54.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:54 smithi055 ceph-mon[24757]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:54.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:54 smithi055 ceph-mon[29498]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:56.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:34:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:34:56.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:56 smithi138 ceph-mon[25673]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:56.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:56 smithi055 ceph-mon[24757]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:56.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:56 smithi055 ceph-mon[29498]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:57.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:34:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:34:57] "GET /metrics HTTP/1.1" 200 208014 "" "Prometheus/2.33.4" 2024-05-20T22:34:58.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:58 smithi138 ceph-mon[25673]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:58.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:58 smithi055 ceph-mon[24757]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:58.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:58 smithi055 ceph-mon[29498]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:34:59.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:34:59 smithi138 ceph-mon[25673]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:59.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:34:59 smithi055 ceph-mon[24757]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:34:59.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:34:59 smithi055 ceph-mon[29498]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:01.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:01 smithi055 ceph-mon[24757]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:35:01.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:01 smithi055 ceph-mon[29498]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:35:01.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:01 smithi138 ceph-mon[25673]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:35:03.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:03 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:03.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:03 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:03.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:03 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:04.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:04 smithi055 ceph-mon[24757]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:04.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:04 smithi055 ceph-mon[29498]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:04.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:04 smithi138 ceph-mon[25673]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:06.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:35:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:35:06.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:06 smithi138 ceph-mon[25673]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:35:06.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:06 smithi055 ceph-mon[24757]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:35:06.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:06 smithi055 ceph-mon[29498]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:35:07.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:35:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:07] "GET /metrics HTTP/1.1" 200 208014 "" "Prometheus/2.33.4" 2024-05-20T22:35:08.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:08 smithi138 ceph-mon[25673]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:35:08.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:08 smithi055 ceph-mon[24757]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:35:08.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:08 smithi055 ceph-mon[29498]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:35:10.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:10 smithi138 ceph-mon[25673]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:10.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:10 smithi055 ceph-mon[24757]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:10.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:10 smithi055 ceph-mon[29498]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:12.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:12 smithi138 ceph-mon[25673]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:12.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:12 smithi055 ceph-mon[24757]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:12.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:12 smithi055 ceph-mon[29498]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:13.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:13 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:13.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:13 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:13.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:13 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:14.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:14 smithi138 ceph-mon[25673]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:14.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:14 smithi055 ceph-mon[24757]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:14.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:14 smithi055 ceph-mon[29498]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:16.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:35:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:35:16.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:16 smithi138 ceph-mon[25673]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:16 smithi055 ceph-mon[24757]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:16.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:16 smithi055 ceph-mon[29498]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:17.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:35:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:17] "GET /metrics HTTP/1.1" 200 208027 "" "Prometheus/2.33.4" 2024-05-20T22:35:18.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:18 smithi138 ceph-mon[25673]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:18.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:18 smithi055 ceph-mon[29498]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:18.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:18 smithi055 ceph-mon[24757]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:19.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:19 smithi138 ceph-mon[25673]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:19.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:19 smithi055 ceph-mon[24757]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:19.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:19 smithi055 ceph-mon[29498]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:21.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:21 smithi055 ceph-mon[24757]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:21.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:21 smithi055 ceph-mon[29498]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:21.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:21 smithi138 ceph-mon[25673]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:23.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:23 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:23.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:23 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:23 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:24 smithi055 ceph-mon[24757]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:24.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:24 smithi055 ceph-mon[29498]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:24.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:24 smithi138 ceph-mon[25673]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:35:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:35:26.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:26 smithi055 ceph-mon[24757]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:26.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:26 smithi055 ceph-mon[29498]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:26 smithi138 ceph-mon[25673]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:27.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:35:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:27] "GET /metrics HTTP/1.1" 200 208032 "" "Prometheus/2.33.4" 2024-05-20T22:35:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:35:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:35:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:35:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:35:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:35:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:35:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:35:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:35:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:35:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:35:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:35:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:35:28.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:28 smithi138 ceph-mon[25673]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:28.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:28 smithi055 ceph-mon[24757]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:28.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:28 smithi055 ceph-mon[29498]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:30.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:30 smithi138 ceph-mon[25673]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:30.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:30 smithi055 ceph-mon[24757]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:30.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:30 smithi055 ceph-mon[29498]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:32.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:32 smithi138 ceph-mon[25673]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:32.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:32 smithi055 ceph-mon[24757]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:32.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:32 smithi055 ceph-mon[29498]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:33.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:33 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:33.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:33 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:33 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:34.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:34 smithi138 ceph-mon[25673]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:34.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:34 smithi055 ceph-mon[29498]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:34.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:34 smithi055 ceph-mon[24757]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:36.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:35:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:35:36.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:36 smithi138 ceph-mon[25673]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:36.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:36.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:36.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:36 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:36.473 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:36 smithi055 ceph-mon[29498]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:36.473 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:36.473 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:36.473 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:36 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:36.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:36 smithi055 ceph-mon[24757]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:36.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:36.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:36.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:36 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:37.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:35:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:37] "GET /metrics HTTP/1.1" 200 208032 "" "Prometheus/2.33.4" 2024-05-20T22:35:37.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:37 smithi138 ceph-mon[25673]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:37.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:37 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:35:37.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:37 smithi055 ceph-mon[24757]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:37.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:37 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:35:37.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:37 smithi055 ceph-mon[29498]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:37.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:37 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:35:39.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:39 smithi138 ceph-mon[25673]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:39.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:39 smithi055 ceph-mon[24757]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:39.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:39 smithi055 ceph-mon[29498]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:41.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:41 smithi138 ceph-mon[25673]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:41.921 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:41 smithi055 ceph-mon[24757]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:41.922 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:41 smithi055 ceph-mon[29498]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:43.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:43 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:43.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:43 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:43.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:43 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:44.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:44 smithi055 ceph-mon[24757]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:44.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:44 smithi055 ceph-mon[29498]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:44.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:44 smithi138 ceph-mon[25673]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:46.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:35:45 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:35:46.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:46 smithi138 ceph-mon[25673]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:46.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:46 smithi055 ceph-mon[29498]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:46.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:46 smithi055 ceph-mon[24757]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:47.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:35:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:47] "GET /metrics HTTP/1.1" 200 208012 "" "Prometheus/2.33.4" 2024-05-20T22:35:48.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:48 smithi138 ceph-mon[25673]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:48.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:48 smithi055 ceph-mon[24757]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:48.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:48 smithi055 ceph-mon[29498]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:50.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:50 smithi138 ceph-mon[25673]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:50.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:50 smithi055 ceph-mon[24757]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:50.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:50 smithi055 ceph-mon[29498]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:52.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:52 smithi138 ceph-mon[25673]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:52.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:52 smithi055 ceph-mon[24757]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:52.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:52 smithi055 ceph-mon[29498]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:53.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:53 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:53.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:53 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:53.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:53 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:35:54.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:54 smithi138 ceph-mon[25673]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:54.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:54 smithi055 ceph-mon[24757]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:54 smithi055 ceph-mon[29498]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:35:56.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:35:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:35:56.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:56 smithi138 ceph-mon[25673]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:56.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:56 smithi055 ceph-mon[24757]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:56.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:56 smithi055 ceph-mon[29498]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:57.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:35:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:35:57] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-05-20T22:35:58.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:35:58 smithi138 ceph-mon[25673]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:58.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:35:58 smithi055 ceph-mon[24757]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:35:58.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:35:58 smithi055 ceph-mon[29498]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:00.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:00 smithi138 ceph-mon[25673]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:00.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:00 smithi055 ceph-mon[24757]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:00.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:00 smithi055 ceph-mon[29498]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:01.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:01 smithi138 ceph-mon[25673]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:01.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:01 smithi055 ceph-mon[24757]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:01.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:01 smithi055 ceph-mon[29498]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:03.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:03 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:03.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:03 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:03.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:03 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:04.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:04 smithi138 ceph-mon[25673]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:04.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:04 smithi055 ceph-mon[24757]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:04.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:04 smithi055 ceph-mon[29498]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:06.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:36:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:36:06.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:06 smithi138 ceph-mon[25673]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:06.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:06 smithi055 ceph-mon[24757]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:06.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:06 smithi055 ceph-mon[29498]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:07.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:36:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:07] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-05-20T22:36:08.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:08 smithi138 ceph-mon[25673]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:08.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:08 smithi055 ceph-mon[29498]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:08.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:08 smithi055 ceph-mon[24757]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:10.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:10 smithi138 ceph-mon[25673]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:10.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:10 smithi055 ceph-mon[29498]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:10.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:10 smithi055 ceph-mon[24757]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:12.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:12 smithi138 ceph-mon[25673]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:12.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:12 smithi055 ceph-mon[24757]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:12.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:12 smithi055 ceph-mon[29498]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:13.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:13 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:13.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:13 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:13.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:13 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:14.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:14 smithi138 ceph-mon[25673]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:14.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:14 smithi055 ceph-mon[24757]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:14.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:14 smithi055 ceph-mon[29498]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:16.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:36:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:36:16.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:16 smithi138 ceph-mon[25673]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:16 smithi055 ceph-mon[24757]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:16.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:16 smithi055 ceph-mon[29498]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:17.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:36:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:17] "GET /metrics HTTP/1.1" 200 207978 "" "Prometheus/2.33.4" 2024-05-20T22:36:18.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:18 smithi138 ceph-mon[25673]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:18.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:18 smithi055 ceph-mon[24757]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:18.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:18 smithi055 ceph-mon[29498]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:20.118 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-05-20T22:36:20.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:20 smithi055 ceph-mon[24757]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:20.439 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:20 smithi055 ceph-mon[29498]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:20.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:20 smithi138 ceph-mon[25673]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:21.573 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:21 smithi055 ceph-mon[24757]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:21.573 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:21 smithi055 ceph-mon[29498]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:21.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:21 smithi138 ceph-mon[25673]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:22.872 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-05-20T22:36:23.449 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:23 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:23.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:23 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:23 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:24 smithi055 ceph-mon[24757]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:24 smithi055 ceph-mon[29498]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:24.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:24 smithi138 ceph-mon[25673]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:26.059 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph config set global log_to_journald false --force' 2024-05-20T22:36:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:36:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:36:26.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:26 smithi055 ceph-mon[24757]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:26.381 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:26 smithi055 ceph-mon[29498]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:26 smithi138 ceph-mon[25673]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:27.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:36:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:27] "GET /metrics HTTP/1.1" 200 207969 "" "Prometheus/2.33.4" 2024-05-20T22:36:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:36:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:36:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:36:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:36:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:36:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:36:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:36:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:36:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:36:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:36:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:36:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:36:28.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:28 smithi055 ceph-mon[24757]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:28.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:28 smithi055 ceph-mon[29498]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:28.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:28 smithi138 ceph-mon[25673]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:28.853 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T22:36:30.405 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:30 smithi055 ceph-mon[24757]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:30.405 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:30 smithi055 ceph-mon[29498]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:30.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:30 smithi138 ceph-mon[25673]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:30.810 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:36:30.810 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (3m) 2m ago 3m 15.1M - ba2b418f427c 37600d6e6b2f 2024-05-20T22:36:30.810 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (3m) 2m ago 3m 45.0M - 8.3.5 dad864ee21e9 93d02684b943 2024-05-20T22:36:30.810 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (2m) 2m ago 2m 68.0M - 3.5 e1d6a67b021e 5cea51176d33 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443 running (8m) 2m ago 8m 410M - 17.2.0 e1d6a67b021e 6b6d8b70e257 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:9283 running (10m) 2m ago 10m 454M - 17.2.0 e1d6a67b021e 70d348a53fb9 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (10m) 2m ago 10m 46.8M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (8m) 2m ago 8m 40.2M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (8m) 2m ago 8m 39.2M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (4m) 2m ago 4m 16.3M - 1dbe0e931976 08e55b1e4140 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (4m) 2m ago 4m 13.7M - 1dbe0e931976 8a4a20d56038 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (8m) 2m ago 8m 48.2M 4017M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (7m) 2m ago 7m 47.5M 4017M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (7m) 2m ago 7m 46.7M 4017M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (7m) 2m ago 6m 45.3M 4017M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (6m) 2m ago 6m 45.7M 4272M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:36:30.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (6m) 2m ago 6m 43.5M 4272M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:36:30.812 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (5m) 2m ago 5m 42.4M 4272M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:36:30.812 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (5m) 2m ago 5m 44.4M 4272M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:36:30.812 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (3m) 2m ago 4m 43.8M - 514e6a882f6e 1c5c1e535c2d 2024-05-20T22:36:30.812 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (3m) 2m ago 3m 68.5M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:36:30.812 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (3m) 2m ago 3m 71.3M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:36:31.442 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions' 2024-05-20T22:36:32.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:32 smithi138 ceph-mon[25673]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:32.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:32 smithi138 ceph-mon[25673]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:32.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:32 smithi055 ceph-mon[24757]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:32.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:32 smithi055 ceph-mon[24757]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:32.648 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:32 smithi055 ceph-mon[29498]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:32.648 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:32 smithi055 ceph-mon[29498]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:33.398 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:36:33.398 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:36:33.398 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:36:33.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:36:33.399 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-05-20T22:36:33.400 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:36:33.400 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:36:33.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:33 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:33.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:33 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:33.714 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:33 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:34.030 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph -s' 2024-05-20T22:36:34.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:34 smithi138 ceph-mon[25673]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:34.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:34 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1661565255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:36:34.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:34 smithi055 ceph-mon[24757]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:34.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:34 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1661565255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:36:34.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:34 smithi055 ceph-mon[29498]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:34.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:34 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1661565255' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:36:35.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:35 smithi138 ceph-mon[25673]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:35.617 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:35 smithi055 ceph-mon[24757]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:35.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:35 smithi055 ceph-mon[29498]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:36.105 INFO:teuthology.orchestra.run.smithi055.stdout: cluster: 2024-05-20T22:36:36.105 INFO:teuthology.orchestra.run.smithi055.stdout: id: dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:36:36.105 INFO:teuthology.orchestra.run.smithi055.stdout: health: HEALTH_OK 2024-05-20T22:36:36.105 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:36:36.105 INFO:teuthology.orchestra.run.smithi055.stdout: services: 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: mgr: y(active, since 5m), standbys: x 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: data: 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: pools: 6 pools, 161 pgs 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: objects: 209 objects, 585 KiB 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: pgs: 161 active+clean 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:36:36.106 INFO:teuthology.orchestra.run.smithi055.stdout: io: 2024-05-20T22:36:36.107 INFO:teuthology.orchestra.run.smithi055.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-20T22:36:36.107 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:36:36.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:36:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:36:36.419 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:36 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3996973603' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:36:36.420 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:36 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3996973603' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:36:36.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:36 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3996973603' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:36:36.722 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ls' 2024-05-20T22:36:37.332 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:36:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:37] "GET /metrics HTTP/1.1" 200 207969 "" "Prometheus/2.33.4" 2024-05-20T22:36:37.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[24757]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:37.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:36:37.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:36:37.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:36:37.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:36:37.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[29498]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:37.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:36:37.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:36:37.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:36:37.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:37 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:36:37.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:37 smithi138 ceph-mon[25673]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:37.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:37 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:36:37.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:37 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:36:37.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:37 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:36:37.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:37 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:36:38.721 INFO:teuthology.orchestra.run.smithi055.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-20T22:36:38.721 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi055=a;count:1 2024-05-20T22:36:38.721 INFO:teuthology.orchestra.run.smithi055.stdout:grafana ?:3000 1/1 3m ago 5m smithi138=a;count:1 2024-05-20T22:36:38.721 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-05-20T22:36:38.721 INFO:teuthology.orchestra.run.smithi055.stdout:mgr 2/2 3m ago 8m smithi055=y;smithi138=x;count:2 2024-05-20T22:36:38.722 INFO:teuthology.orchestra.run.smithi055.stdout:mon 3/3 3m ago 9m smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi138:172.21.15.138=b;count:3 2024-05-20T22:36:38.722 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi055=a;smithi138=b;count:2 2024-05-20T22:36:38.722 INFO:teuthology.orchestra.run.smithi055.stdout:osd 8 3m ago - 2024-05-20T22:36:38.722 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus ?:9095 1/1 3m ago 5m smithi138=a;count:1 2024-05-20T22:36:38.722 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-05-20T22:36:39.381 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-20T22:36:39.699 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:39 smithi055 ceph-mon[24757]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:39.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:39 smithi055 ceph-mon[24757]: from='client.24874 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:39.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:39 smithi055 ceph-mon[29498]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:39.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:39 smithi055 ceph-mon[29498]: from='client.24874 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:39.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:39 smithi138 ceph-mon[25673]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:39.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:39 smithi138 ceph-mon[25673]: from='client.24874 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:41.655 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled to redeploy mgr.x on host 'smithi138' 2024-05-20T22:36:41.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:41 smithi055 ceph-mon[24757]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:41.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:41 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2835033352' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:36:41.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:41 smithi055 ceph-mon[29498]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:41.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:41 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2835033352' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:36:41.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:41 smithi138 ceph-mon[25673]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:41.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:41 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2835033352' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:36:42.267 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps --refresh' 2024-05-20T22:36:42.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[29498]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:42.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:36:42.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[29498]: Schedule redeploy daemon mgr.x 2024-05-20T22:36:42.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:36:42.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:42.942 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[24757]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:42.943 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:36:42.943 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[24757]: Schedule redeploy daemon mgr.x 2024-05-20T22:36:42.943 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:36:42.943 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:42 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:43.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:42 smithi138 ceph-mon[25673]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:43.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:42 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:36:43.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:42 smithi138 ceph-mon[25673]: Schedule redeploy daemon mgr.x 2024-05-20T22:36:43.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:42 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:36:43.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:42 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:44.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:44 smithi055 ceph-mon[24757]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:44.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:44 smithi055 ceph-mon[29498]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:44.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:44 smithi138 ceph-mon[25673]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (3m) 3m ago 4m 15.1M - ba2b418f427c 37600d6e6b2f 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (3m) 3m ago 3m 45.0M - 8.3.5 dad864ee21e9 93d02684b943 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (3m) 3m ago 3m 68.0M - 3.5 e1d6a67b021e 5cea51176d33 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 6b6d8b70e257 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e 70d348a53fb9 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (10m) 3m ago 10m 46.8M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (9m) 3m ago 9m 40.2M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (9m) 3m ago 9m 39.2M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:36:44.704 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (4m) 3m ago 4m 16.3M - 1dbe0e931976 08e55b1e4140 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (4m) 3m ago 4m 13.7M - 1dbe0e931976 8a4a20d56038 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (8m) 3m ago 8m 48.2M 4017M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (8m) 3m ago 8m 47.5M 4017M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (7m) 3m ago 7m 46.7M 4017M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (7m) 3m ago 7m 45.3M 4017M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (6m) 3m ago 6m 45.7M 4272M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (6m) 3m ago 6m 43.5M 4272M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (6m) 3m ago 6m 42.4M 4272M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (5m) 3m ago 5m 44.4M 4272M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (3m) 3m ago 5m 43.8M - 514e6a882f6e 1c5c1e535c2d 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (3m) 3m ago 3m 68.5M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:36:44.705 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (3m) 3m ago 3m 71.3M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:36:45.466 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'sleep 180' 2024-05-20T22:36:46.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:36:45 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:36:46.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:46 smithi055 ceph-mon[24757]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:46.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:46 smithi055 ceph-mon[24757]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:46.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:46 smithi055 ceph-mon[29498]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:46.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:46 smithi055 ceph-mon[29498]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:46.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:46 smithi138 ceph-mon[25673]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:46.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:46 smithi138 ceph-mon[25673]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:47.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:36:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:47] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-05-20T22:36:48.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:48 smithi138 ceph-mon[25673]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:48.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:48 smithi055 ceph-mon[24757]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:48 smithi055 ceph-mon[29498]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:50.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:50 smithi138 ceph-mon[25673]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:50.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:50 smithi055 ceph-mon[24757]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:50.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:50 smithi055 ceph-mon[29498]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:52.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:52 smithi138 ceph-mon[25673]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:52.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:52 smithi055 ceph-mon[29498]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:52.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:52 smithi055 ceph-mon[24757]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:53.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:53 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:53.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:53 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:53.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:53 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:36:54.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:54 smithi138 ceph-mon[25673]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:54.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:54 smithi055 ceph-mon[24757]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:54 smithi055 ceph-mon[29498]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:36:55.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:55 smithi138 ceph-mon[25673]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:55.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:55 smithi055 ceph-mon[24757]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:55 smithi055 ceph-mon[29498]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:56.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:36:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:36:57.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:36:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:36:57] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-05-20T22:36:58.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:36:58 smithi138 ceph-mon[25673]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:58.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:36:58 smithi055 ceph-mon[24757]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:36:58.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:36:58 smithi055 ceph-mon[29498]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:00.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:00 smithi138 ceph-mon[25673]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:00.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:00 smithi055 ceph-mon[24757]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:00.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:00 smithi055 ceph-mon[29498]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:02.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:02 smithi138 ceph-mon[25673]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:02.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:02 smithi055 ceph-mon[24757]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:02.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:02 smithi055 ceph-mon[29498]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:03.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:03 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:03.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:03 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:03.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:03 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:04.572 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:04 smithi055 ceph-mon[24757]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:04.572 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:04 smithi055 ceph-mon[29498]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:04.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:04 smithi138 ceph-mon[25673]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:06.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:37:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:37:06.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:06 smithi138 ceph-mon[25673]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:06.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:06 smithi055 ceph-mon[24757]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:06.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:06 smithi055 ceph-mon[29498]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:07.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:37:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:07] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-05-20T22:37:08.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:08 smithi138 ceph-mon[25673]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:08.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:08 smithi055 ceph-mon[24757]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:08.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:08 smithi055 ceph-mon[29498]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:10.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:10 smithi138 ceph-mon[25673]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:10.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:10 smithi055 ceph-mon[24757]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:10.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:10 smithi055 ceph-mon[29498]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:12.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:12 smithi138 ceph-mon[25673]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:12.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:12 smithi055 ceph-mon[24757]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:12.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:12 smithi055 ceph-mon[29498]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:13.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:13 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:13.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:13 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:13.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:13 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:14.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:14 smithi138 ceph-mon[25673]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:14.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:14 smithi055 ceph-mon[24757]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:14.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:14 smithi055 ceph-mon[29498]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:15.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:15 smithi138 ceph-mon[25673]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:15.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:15 smithi055 ceph-mon[24757]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:15.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:15 smithi055 ceph-mon[29498]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:16.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:37:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:37:17.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:37:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:17] "GET /metrics HTTP/1.1" 200 207971 "" "Prometheus/2.33.4" 2024-05-20T22:37:18.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:18 smithi138 ceph-mon[25673]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:18.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:18 smithi055 ceph-mon[24757]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:18.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:18 smithi055 ceph-mon[29498]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:20.558 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:20 smithi055 ceph-mon[24757]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:20.559 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:20 smithi055 ceph-mon[29498]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:20.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:20 smithi138 ceph-mon[25673]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:21.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:21 smithi138 ceph-mon[25673]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:21.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:21 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:21.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:21 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:37:21.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:21 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:37:21.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:21 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:21.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:21 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:21.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[29498]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:21.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:21.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:37:21.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:37:21.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:21.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:21.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[24757]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:21.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:21.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:37:21.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:37:21.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:21.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:21 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:23.512 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:23 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:23.551 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:23 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:23.552 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:23 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:24.586 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:24 smithi055 ceph-mon[24757]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:24.586 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:24 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:24.587 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:24 smithi055 ceph-mon[29498]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:24.587 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:24 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:24.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:24 smithi138 ceph-mon[25673]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:24.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:24 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:37:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:37:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:26 smithi138 ceph-mon[25673]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:26 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:26 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:26 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:37:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:26 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:37:26.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:26 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:37:26.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:26 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:37:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[24757]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:37:26.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:37:26.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:37:26.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:37:26.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[29498]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:26.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:26.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:37:26.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:37:26.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:37:26.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:37:26.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:26 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:37:27.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:37:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:27] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-05-20T22:37:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[24757]: Deploying daemon mgr.x on smithi138 2024-05-20T22:37:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:37:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:37:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:37:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:37:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[29498]: Deploying daemon mgr.x on smithi138 2024-05-20T22:37:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:37:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:37:27.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:37:27.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:37:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:27 smithi138 ceph-mon[25673]: Deploying daemon mgr.x on smithi138 2024-05-20T22:37:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:37:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:37:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:37:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:37:28.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:28 smithi138 ceph-mon[25673]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:28.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:28 smithi055 ceph-mon[24757]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:28.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:28 smithi055 ceph-mon[29498]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:30.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:30 smithi138 ceph-mon[25673]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:30.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:30 smithi055 ceph-mon[24757]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:30.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:30 smithi055 ceph-mon[29498]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:32.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:32 smithi138 ceph-mon[25673]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:32.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:32 smithi055 ceph-mon[24757]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:32.714 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:32 smithi055 ceph-mon[29498]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:33.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:33 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:33 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:33.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:33 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:33.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:37:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:37:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:37:33.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:37:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:37:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:37:34.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:34 smithi138 ceph-mon[25673]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:34.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:34 smithi055 ceph-mon[24757]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:34.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:34 smithi055 ceph-mon[29498]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:35.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:35 smithi138 ceph-mon[25673]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:35.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:35 smithi055 ceph-mon[24757]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:35.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:35 smithi055 ceph-mon[29498]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:36.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:37:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:37:37.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:37:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:37] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-05-20T22:37:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:38 smithi138 ceph-mon[25673]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:38.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:38 smithi055 ceph-mon[24757]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:38 smithi055 ceph-mon[29498]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:40.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:40 smithi138 ceph-mon[25673]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:40.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:40 smithi055 ceph-mon[24757]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:40.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:40 smithi055 ceph-mon[29498]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:42.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:42 smithi138 ceph-mon[25673]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:42.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:42 smithi055 ceph-mon[24757]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:42.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:42 smithi055 ceph-mon[29498]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:43.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:43 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:43 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:43.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:43 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:43.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:37:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:37:43.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:37:43.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:37:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:37:43.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:37:43.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:37:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:37:43.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:37:44.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:44 smithi138 ceph-mon[25673]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:44.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:44 smithi055 ceph-mon[24757]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:45.793 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:44 smithi055 ceph-mon[29498]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:46.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:37:45 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:37:46.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:46 smithi138 ceph-mon[25673]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:46.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:46 smithi055 ceph-mon[24757]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:46.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:46 smithi055 ceph-mon[29498]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:47.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:37:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:47] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-05-20T22:37:48.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:48 smithi138 ceph-mon[25673]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:48.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:48 smithi055 ceph-mon[24757]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:48 smithi055 ceph-mon[29498]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:49.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:49 smithi138 ceph-mon[25673]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:49.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:49 smithi055 ceph-mon[24757]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:49.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:49 smithi055 ceph-mon[29498]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:51.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:51 smithi138 ceph-mon[25673]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:51.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:51 smithi055 ceph-mon[24757]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:51.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:51 smithi055 ceph-mon[29498]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:53.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:53 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:53.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:53 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:53.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:53 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:37:53.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:37:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:37:53.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:37:53.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:37:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:37:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:37:53.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:37:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:37:53.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:37:54.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:54 smithi138 ceph-mon[25673]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:54.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:54 smithi055 ceph-mon[24757]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:54 smithi055 ceph-mon[29498]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:37:56.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:37:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[26847]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:37:56.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:56 smithi138 ceph-mon[25673]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:56.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:56 smithi055 ceph-mon[24757]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:56.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:56 smithi055 ceph-mon[29498]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:57.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:37:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:37:57] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-05-20T22:37:58.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:37:58 smithi138 ceph-mon[25673]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:58.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:37:58 smithi055 ceph-mon[24757]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:37:58.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:37:58 smithi055 ceph-mon[29498]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:00.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:00 smithi138 ceph-mon[25673]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:00.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:00 smithi138 systemd[1]: Stopping Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:38:00.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:00 smithi055 ceph-mon[24757]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:00.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:00 smithi055 ceph-mon[29498]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:00.758 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:00 smithi138 bash[45601]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr.x" found: no such container 2024-05-20T22:38:00.758 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:00 smithi138 podman[45612]: 2024-05-20 22:38:00.509902508 +0000 UTC m=+0.126482161 container died 6b6d8b70e257f3ed566c78020e708b2fbd4386a0a5efe97f362d008ef6e82dce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, name=centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-20T22:38:01.026 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:00 smithi138 podman[45612]: 2024-05-20 22:38:00.839159528 +0000 UTC m=+0.455739172 container cleanup 6b6d8b70e257f3ed566c78020e708b2fbd4386a0a5efe97f362d008ef6e82dce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, ceph=True, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, release=754, maintainer=Guillaume Abrioux , io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.k8s.display-name=CentOS Stream 8) 2024-05-20T22:38:01.026 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:00 smithi138 bash[45612]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x 2024-05-20T22:38:01.026 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:00 smithi138 bash[45640]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr.x" found: no such container 2024-05-20T22:38:01.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 podman[45626]: 2024-05-20 22:38:01.032729771 +0000 UTC m=+0.518308481 container remove 6b6d8b70e257f3ed566c78020e708b2fbd4386a0a5efe97f362d008ef6e82dce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, release=754, ceph=True, name=centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=) 2024-05-20T22:38:01.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-20T22:38:01.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-05-20T22:38:01.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 systemd[1]: Stopped Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:38:01.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.x.service: Consumed 17.424s CPU time. 2024-05-20T22:38:01.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 systemd[1]: Starting Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:38:01.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 podman[45742]: 2024-05-20 22:38:01.336900532 +0000 UTC m=+0.017222567 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:38:01.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 podman[45742]: 2024-05-20 22:38:01.468175516 +0000 UTC m=+0.148497547 container create 0639e81ef3c82f2216fc95f2a4a1adc12a9db69215512a368b85fba6b7608357 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:38:01.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 podman[45742]: 2024-05-20 22:38:01.67101427 +0000 UTC m=+0.351336308 container init 0639e81ef3c82f2216fc95f2a4a1adc12a9db69215512a368b85fba6b7608357 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514) 2024-05-20T22:38:01.692 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 podman[45742]: 2024-05-20 22:38:01.674485308 +0000 UTC m=+0.354807339 container start 0639e81ef3c82f2216fc95f2a4a1adc12a9db69215512a368b85fba6b7608357 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5) 2024-05-20T22:38:02.024 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 ceph-mgr[45761]: -- 172.21.15.138:0/1678134950 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5578de75a5a0 con 0x5578de72ec00 2024-05-20T22:38:02.024 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 bash[45742]: 0639e81ef3c82f2216fc95f2a4a1adc12a9db69215512a368b85fba6b7608357 2024-05-20T22:38:02.024 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 systemd[1]: Started Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:38:02.024 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:01.834+0000 7f48dbc75200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:38:02.024 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:02 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:02.029+0000 7f48dbc75200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:38:02.298 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:02 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:02.153+0000 7f48dbc75200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:38:02.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[24757]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:02.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:02.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:02.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:38:02.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:38:02.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:38:02.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[29498]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:02.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:02.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:02.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:38:02.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:38:02.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:02 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:38:02.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:02 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:02.486+0000 7f48dbc75200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:38:02.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:02 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:02.575+0000 7f48dbc75200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:38:02.570 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:02 smithi138 ceph-mon[25673]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:02.570 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:02 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:02.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:02 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:02.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:02 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:38:02.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:02 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:38:02.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:02 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:38:03.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:03 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:03.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:03 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:03.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:03 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:03.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:38:03.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:38:03.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:03.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T22:38:03.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:38:04.141 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:03 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:03.872+0000 7f48dbc75200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:38:04.141 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:04.059+0000 7f48dbc75200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:38:04.570 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:04 smithi138 ceph-mon[25673]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:04.570 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:04.241+0000 7f48dbc75200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:38:04.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:04 smithi055 ceph-mon[24757]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:04.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:04 smithi055 ceph-mon[29498]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:04.853 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:04.575+0000 7f48dbc75200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:38:04.853 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:04.671+0000 7f48dbc75200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:38:05.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:04.885+0000 7f48dbc75200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:38:05.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:05.087+0000 7f48dbc75200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:38:06.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:05.734+0000 7f48dbc75200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:38:06.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:06 smithi138 ceph-mon[25673]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:06.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:06 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:06.491 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:06 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:06.492 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:06.191+0000 7f48dbc75200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:38:06.492 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:06.290+0000 7f48dbc75200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:38:06.492 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:06.497+0000 7f48dbc75200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T22:38:06.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:06 smithi055 ceph-mon[24757]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:06.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:06 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:06.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:06 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:06.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:06 smithi055 ceph-mon[29498]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:06.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:06 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:06.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:06 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:38:07.357 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:07.024+0000 7f48dbc75200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:38:07.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:07.118+0000 7f48dbc75200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:38:07.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:38:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:07] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-05-20T22:38:07.721 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:07.378+0000 7f48dbc75200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:38:07.721 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:07.512+0000 7f48dbc75200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:38:07.721 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:07.609+0000 7f48dbc75200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:38:07.722 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:07.727+0000 7f48dbc75200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:38:08.324 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:08.033+0000 7f48dbc75200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:38:08.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:08 smithi138 ceph-mon[25673]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:08.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:08 smithi055 ceph-mon[24757]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:08.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:08 smithi055 ceph-mon[29498]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:08.964 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:08.642+0000 7f48dbc75200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:38:08.964 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:08.863+0000 7f48dbc75200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:38:08.964 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:38:08] ENGINE Bus STARTING 2024-05-20T22:38:08.964 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: CherryPy Checker: 2024-05-20T22:38:08.964 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: The Application mounted at '' has an empty config. 2024-05-20T22:38:08.964 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:38:08.964 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:38:08] ENGINE Serving on http://:::9283 2024-05-20T22:38:09.608 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:38:08] ENGINE Bus STARTED 2024-05-20T22:38:09.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:09 smithi138 ceph-mon[25673]: Standby manager daemon x restarted 2024-05-20T22:38:09.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:09 smithi138 ceph-mon[25673]: Standby manager daemon x started 2024-05-20T22:38:09.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:09 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:38:09.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:09 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:38:09.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:09 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:38:09.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:09 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:38:09.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[24757]: Standby manager daemon x restarted 2024-05-20T22:38:09.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[24757]: Standby manager daemon x started 2024-05-20T22:38:09.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:38:09.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:38:09.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:38:09.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:38:09.656 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[29498]: Standby manager daemon x restarted 2024-05-20T22:38:09.656 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[29498]: Standby manager daemon x started 2024-05-20T22:38:09.656 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:38:09.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:38:09.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:38:09.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:09 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:38:09.985 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:09.662Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:38:10.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:10 smithi138 ceph-mon[25673]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:10.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:10 smithi138 ceph-mon[25673]: mgrmap e22: y(active, since 6m), standbys: x 2024-05-20T22:38:10.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:10 smithi055 ceph-mon[29498]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:10.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:10 smithi055 ceph-mon[29498]: mgrmap e22: y(active, since 6m), standbys: x 2024-05-20T22:38:10.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:10 smithi055 ceph-mon[24757]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:10.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:10 smithi055 ceph-mon[24757]: mgrmap e22: y(active, since 6m), standbys: x 2024-05-20T22:38:11.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:11 smithi138 ceph-mon[25673]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:11.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:11 smithi055 ceph-mon[24757]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:11.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:11 smithi055 ceph-mon[29498]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:13.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:13 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:13.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:13 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:13.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:13 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:13.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:38:13.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:38:13.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:38:13.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:38:14.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:14 smithi138 ceph-mon[25673]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:14.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:14 smithi055 ceph-mon[29498]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:14.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:14 smithi055 ceph-mon[24757]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:16.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:38:16.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:16 smithi138 ceph-mon[25673]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:16.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:16 smithi055 ceph-mon[29498]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:16 smithi055 ceph-mon[24757]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:17.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:38:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:17] "GET /metrics HTTP/1.1" 200 207989 "" "Prometheus/2.33.4" 2024-05-20T22:38:18.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:18 smithi138 ceph-mon[25673]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:18.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:18 smithi055 ceph-mon[24757]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:18.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:18 smithi055 ceph-mon[29498]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:20.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:20 smithi138 ceph-mon[25673]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:20 smithi055 ceph-mon[24757]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:20.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:20 smithi055 ceph-mon[29498]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:22.589 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:22 smithi055 ceph-mon[24757]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:22.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:22 smithi055 ceph-mon[29498]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:22.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:22 smithi138 ceph-mon[25673]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:23.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:23 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:23 smithi138 ceph-mon[25673]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:23.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:23 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:23.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:23 smithi055 ceph-mon[29498]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:23.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:23 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:23.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:23 smithi055 ceph-mon[24757]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:23.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:38:23.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:38:23.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:38:23.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:23.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:38:25.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:25 smithi138 ceph-mon[25673]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:25.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:25 smithi055 ceph-mon[29498]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:25.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:25 smithi055 ceph-mon[24757]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:38:26.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:26 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:38:26.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:26 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:38:26.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:26 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:38:26.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:26 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:38:26.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:26 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:38:26.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:26 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:38:26.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:26 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:38:26.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:26 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:38:26.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:26 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:38:26.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:26 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:38:26.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:26 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:38:26.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:26 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:38:27.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:38:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:27] "GET /metrics HTTP/1.1" 200 207994 "" "Prometheus/2.33.4" 2024-05-20T22:38:28.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:28 smithi138 ceph-mon[25673]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:28.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:28 smithi055 ceph-mon[29498]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:28.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:28 smithi055 ceph-mon[24757]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:30.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:30 smithi138 ceph-mon[25673]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:30.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:30 smithi055 ceph-mon[24757]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:30.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:30 smithi055 ceph-mon[29498]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:32.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:32 smithi055 ceph-mon[24757]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:32.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:32 smithi055 ceph-mon[29498]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:32.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:32 smithi138 ceph-mon[25673]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:33 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:33.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:33 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:33.627 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:33 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:33.627 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:38:33.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:38:33.627 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:38:33.628 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:38:34.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:34 smithi138 ceph-mon[25673]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:34.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:34 smithi055 ceph-mon[24757]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:34.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:34 smithi055 ceph-mon[29498]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:36.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:38:36.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:36 smithi138 ceph-mon[25673]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:36.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:36 smithi055 ceph-mon[24757]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:36.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:36 smithi055 ceph-mon[29498]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:37.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:38:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:37] "GET /metrics HTTP/1.1" 200 207994 "" "Prometheus/2.33.4" 2024-05-20T22:38:38.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:38 smithi138 ceph-mon[25673]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:38.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:38 smithi055 ceph-mon[24757]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:38 smithi055 ceph-mon[29498]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:40.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:40 smithi138 ceph-mon[25673]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:40.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:40 smithi055 ceph-mon[24757]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:40.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:40 smithi055 ceph-mon[29498]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:42.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:42 smithi055 ceph-mon[24757]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:42.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:42 smithi055 ceph-mon[29498]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:42.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:42 smithi138 ceph-mon[25673]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:43.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:43 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:43 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:43.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:43 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:43.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:38:43.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:38:43.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:43.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:38:43.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:43.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:38:44.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:44 smithi138 ceph-mon[25673]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:44.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:44 smithi055 ceph-mon[24757]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:44.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:44 smithi055 ceph-mon[29498]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:45.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:45 smithi055 ceph-mon[24757]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:45.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:45 smithi055 ceph-mon[29498]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:45.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:45 smithi138 ceph-mon[25673]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:46.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:45 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:38:47.487 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:38:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:47] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-05-20T22:38:48.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:48 smithi138 ceph-mon[25673]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:48.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:48 smithi055 ceph-mon[24757]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:48 smithi055 ceph-mon[29498]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:50.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:50 smithi138 ceph-mon[25673]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:50.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:50 smithi055 ceph-mon[24757]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:50.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:50 smithi055 ceph-mon[29498]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:52.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:52 smithi055 ceph-mon[24757]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:52.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:52 smithi055 ceph-mon[29498]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:52.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:52 smithi138 ceph-mon[25673]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:53.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:53 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:53.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:53 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:53.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:53 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:38:53.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:38:53.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:38:53.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:53.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:38:53.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:38:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:38:53.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:38:54.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:54 smithi138 ceph-mon[25673]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:54.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:54 smithi055 ceph-mon[24757]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:54 smithi055 ceph-mon[29498]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:56.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:38:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:38:56.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:56 smithi138 ceph-mon[25673]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:56.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:56 smithi055 ceph-mon[24757]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:56.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:56 smithi055 ceph-mon[29498]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:57.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:38:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:38:57] "GET /metrics HTTP/1.1" 200 207985 "" "Prometheus/2.33.4" 2024-05-20T22:38:58.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:58 smithi138 ceph-mon[25673]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:58.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:58 smithi055 ceph-mon[24757]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:58.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:58 smithi055 ceph-mon[29498]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:38:59.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:38:59 smithi055 ceph-mon[24757]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:59.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:38:59 smithi055 ceph-mon[29498]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:38:59.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:38:59 smithi138 ceph-mon[25673]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:01.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:01 smithi138 ceph-mon[25673]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:01.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:01 smithi055 ceph-mon[24757]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:01.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:01 smithi055 ceph-mon[29498]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:03.610 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:03 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:03.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:03 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:03.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:03 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:03.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:39:03.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:39:03.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:03.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:39:03.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:03.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:39:04.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:04 smithi138 ceph-mon[25673]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:04.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:04 smithi055 ceph-mon[24757]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:04.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:04 smithi055 ceph-mon[29498]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:06.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:39:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:39:06.474 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:06 smithi138 ceph-mon[25673]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:06.474 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:06 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:39:06.474 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:06 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:39:06.474 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:06 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:39:06.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:06 smithi055 ceph-mon[24757]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:06.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:06 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:39:06.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:06 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:39:06.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:06 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:39:06.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:06 smithi055 ceph-mon[29498]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:06.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:06 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:39:06.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:06 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:39:06.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:06 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:39:07.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:39:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:07] "GET /metrics HTTP/1.1" 200 207985 "" "Prometheus/2.33.4" 2024-05-20T22:39:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:07 smithi138 ceph-mon[25673]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:07.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:07 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:39:07.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:07 smithi055 ceph-mon[24757]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:07.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:07 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:39:07.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:07 smithi055 ceph-mon[29498]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:07.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:07 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' 2024-05-20T22:39:09.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:09 smithi138 ceph-mon[25673]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:09.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:09 smithi055 ceph-mon[24757]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:09.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:09 smithi055 ceph-mon[29498]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:11.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:11 smithi138 ceph-mon[25673]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:11.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:11 smithi055 ceph-mon[24757]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:11.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:11 smithi055 ceph-mon[29498]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:13.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:13 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:13.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:13 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:13.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:13 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:13.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:39:13.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:39:13.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:13.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:39:13.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:39:14.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:14 smithi138 ceph-mon[25673]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:14.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:14 smithi055 ceph-mon[24757]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:14.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:14 smithi055 ceph-mon[29498]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:16.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:39:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:39:16.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:16 smithi138 ceph-mon[25673]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:16 smithi055 ceph-mon[24757]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:16.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:16 smithi055 ceph-mon[29498]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:17.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:39:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:17] "GET /metrics HTTP/1.1" 200 207986 "" "Prometheus/2.33.4" 2024-05-20T22:39:18.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:18 smithi138 ceph-mon[25673]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:18.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:18 smithi055 ceph-mon[24757]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:18.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:18 smithi055 ceph-mon[29498]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:20.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:20 smithi138 ceph-mon[25673]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:20 smithi055 ceph-mon[24757]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:20.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:20 smithi055 ceph-mon[29498]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:21.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:21 smithi055 ceph-mon[24757]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:21.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:21 smithi055 ceph-mon[29498]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:21.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:21 smithi138 ceph-mon[25673]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:23.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:23 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:23.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:23 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:23.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:23 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:23.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:39:23.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:39:23.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:23.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:39:23.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:39:24.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:24 smithi138 ceph-mon[25673]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:24.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:24 smithi055 ceph-mon[24757]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:24.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:24 smithi055 ceph-mon[29498]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:26.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:39:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:39:26.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:26 smithi138 ceph-mon[25673]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:26 smithi055 ceph-mon[24757]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:26.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:26 smithi055 ceph-mon[29498]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:27.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:39:27.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:39:27.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:27 smithi055 ceph-mon[29498]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:39:27.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:27 smithi055 ceph-mon[29498]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:39:27.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:39:27.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:39:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:27 smithi055 ceph-mon[24757]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:39:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:27 smithi055 ceph-mon[24757]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:39:27.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:39:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:27] "GET /metrics HTTP/1.1" 200 207996 "" "Prometheus/2.33.4" 2024-05-20T22:39:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:39:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:39:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:27 smithi138 ceph-mon[25673]: from='mgr.14433 172.21.15.55:0/3869926098' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:39:27.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:27 smithi138 ceph-mon[25673]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:39:28.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:28 smithi138 ceph-mon[25673]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:28.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:28 smithi055 ceph-mon[24757]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:28.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:28 smithi055 ceph-mon[29498]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:30.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:30 smithi138 ceph-mon[25673]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:30.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:30 smithi055 ceph-mon[24757]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:30.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:30 smithi055 ceph-mon[29498]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:32.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:32 smithi055 ceph-mon[24757]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:32.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:32 smithi055 ceph-mon[29498]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:32.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:32 smithi138 ceph-mon[25673]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:33.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:33 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:33 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:33.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:33 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:33.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:39:33.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:39:33.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:33.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:39:33.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:33.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:39:34.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:34 smithi138 ceph-mon[25673]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:34.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:34 smithi055 ceph-mon[24757]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:34.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:34 smithi055 ceph-mon[29498]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:36.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:39:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:39:36.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:36 smithi138 ceph-mon[25673]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:36.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:36 smithi055 ceph-mon[24757]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:36.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:36 smithi055 ceph-mon[29498]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:37.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:39:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:37] "GET /metrics HTTP/1.1" 200 207996 "" "Prometheus/2.33.4" 2024-05-20T22:39:38.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:38 smithi138 ceph-mon[25673]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:38.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:38 smithi055 ceph-mon[24757]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:38 smithi055 ceph-mon[29498]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:40.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:40 smithi138 ceph-mon[25673]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:40.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:40 smithi055 ceph-mon[24757]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:40.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:40 smithi055 ceph-mon[29498]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:42.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:42 smithi055 ceph-mon[24757]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:39:42.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:42 smithi055 ceph-mon[29498]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:39:42.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:42 smithi138 ceph-mon[25673]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:39:43.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:43 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:43 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:43.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:43 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:43.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:39:43.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:39:43.737 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:43.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:39:43.737 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:39:44.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:44 smithi138 ceph-mon[25673]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:44.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:44 smithi055 ceph-mon[24757]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:44.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:44 smithi055 ceph-mon[29498]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:46.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:39:45 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:39:46.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:46 smithi138 ceph-mon[25673]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:39:46.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:46 smithi055 ceph-mon[24757]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:39:46.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:46 smithi055 ceph-mon[29498]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:39:47.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:39:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:47] "GET /metrics HTTP/1.1" 200 207986 "" "Prometheus/2.33.4" 2024-05-20T22:39:48.109 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T22:39:48.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:48 smithi138 ceph-mon[25673]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:39:48.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:48 smithi055 ceph-mon[24757]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:39:48.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:48 smithi055 ceph-mon[29498]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:39:50.128 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:39:50.128 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (6m) 2m ago 7m 23.9M - ba2b418f427c 37600d6e6b2f 2024-05-20T22:39:50.128 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (6m) 104s ago 6m 55.1M - 8.3.5 dad864ee21e9 93d02684b943 2024-05-20T22:39:50.128 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (6m) 2m ago 6m 51.2M - 3.5 e1d6a67b021e 5cea51176d33 2024-05-20T22:39:50.128 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283 running (108s) 104s ago 11m 41.8M - 19.0.0-2695-g8a373da3 86045c3f4e75 0639e81ef3c8 2024-05-20T22:39:50.128 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:9283 running (13m) 2m ago 13m 460M - 17.2.0 e1d6a67b021e 70d348a53fb9 2024-05-20T22:39:50.128 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (13m) 2m ago 13m 50.1M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:39:50.128 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (12m) 104s ago 12m 41.3M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:39:50.128 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (12m) 2m ago 12m 42.3M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (8m) 2m ago 7m 16.7M - 1dbe0e931976 08e55b1e4140 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (7m) 104s ago 7m 15.2M - 1dbe0e931976 8a4a20d56038 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (11m) 2m ago 11m 50.9M 4017M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (11m) 2m ago 11m 50.3M 4017M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (10m) 2m ago 10m 49.0M 4017M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (10m) 2m ago 10m 48.5M 4017M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (9m) 104s ago 9m 49.0M 4272M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (9m) 104s ago 9m 47.3M 4272M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (9m) 104s ago 9m 45.9M 4272M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (8m) 104s ago 8m 48.0M 4272M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (6m) 104s ago 8m 64.2M - 514e6a882f6e 1c5c1e535c2d 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (6m) 2m ago 6m 69.5M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:39:50.129 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (6m) 104s ago 6m 72.6M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:39:50.408 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:50 smithi055 ceph-mon[24757]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:50.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:50 smithi055 ceph-mon[29498]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:50.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:50 smithi138 ceph-mon[25673]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:50.738 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions' 2024-05-20T22:39:51.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:51 smithi138 ceph-mon[25673]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:39:51.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:51 smithi138 ceph-mon[25673]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:51.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:51 smithi055 ceph-mon[24757]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:39:51.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:51 smithi055 ceph-mon[24757]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:51.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:51 smithi055 ceph-mon[29498]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:39:51.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:51 smithi055 ceph-mon[29498]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:52.835 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:39:52.835 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:39:52.835 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:39:52.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:39:52.835 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:39:52.835 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-05-20T22:39:52.836 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:39:52.837 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:39:52.837 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:39:53.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:53 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:53.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:53 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/35428448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:39:53.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:53 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:53.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:53 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/35428448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:39:53.534 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph -s' 2024-05-20T22:39:53.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:53 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:39:53.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:53 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/35428448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:39:53.853 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:39:53.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:39:53.853 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:53.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:39:53.853 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:39:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:39:53.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:39:54.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:54 smithi055 ceph-mon[24757]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:54.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:54 smithi055 ceph-mon[29498]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:54.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:54 smithi138 ceph-mon[25673]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:39:55.588 INFO:teuthology.orchestra.run.smithi055.stdout: cluster: 2024-05-20T22:39:55.588 INFO:teuthology.orchestra.run.smithi055.stdout: id: dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: health: HEALTH_OK 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: services: 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: mgr: y(active, since 8m), standbys: x 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: data: 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: pools: 6 pools, 161 pgs 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: objects: 209 objects, 585 KiB 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:39:55.589 INFO:teuthology.orchestra.run.smithi055.stdout: pgs: 161 active+clean 2024-05-20T22:39:55.590 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:39:55.590 INFO:teuthology.orchestra.run.smithi055.stdout: io: 2024-05-20T22:39:55.590 INFO:teuthology.orchestra.run.smithi055.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-20T22:39:55.590 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:39:56.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:39:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:39:56.185 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T22:39:56.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:56 smithi055 ceph-mon[24757]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:56.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:56 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/101668425' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:39:56.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:56 smithi055 ceph-mon[29498]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:56.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:56 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/101668425' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:39:56.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:56 smithi138 ceph-mon[25673]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:56.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:56 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/101668425' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:39:57.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:39:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:39:57] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-05-20T22:39:58.207 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T22:39:58.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:58 smithi055 ceph-mon[24757]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:58.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:39:58 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/66270610' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:39:58.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:58 smithi055 ceph-mon[29498]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:58.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:39:58 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/66270610' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:39:58.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:58 smithi138 ceph-mon[25673]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:39:58.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:39:58 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/66270610' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:39:58.791 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-05-20T22:40:00.554 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:00 smithi055 ceph-mon[24757]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:40:00.554 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:00 smithi055 ceph-mon[24757]: overall HEALTH_OK 2024-05-20T22:40:00.554 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:00 smithi055 ceph-mon[29498]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:40:00.554 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:00 smithi055 ceph-mon[29498]: overall HEALTH_OK 2024-05-20T22:40:00.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:00 smithi138 ceph-mon[25673]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:40:00.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:00 smithi138 ceph-mon[25673]: overall HEALTH_OK 2024-05-20T22:40:00.829 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:40:01.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:01 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/376495003' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:40:01.372 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:01 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/376495003' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:40:01.374 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph mgr fail' 2024-05-20T22:40:01.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:01 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/376495003' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:40:02.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:02 smithi138 ceph-mon[25673]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:40:02.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:02 smithi055 ceph-mon[24757]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:40:02.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:02 smithi055 ceph-mon[29498]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:40:03.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:03 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:03.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:03 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:03.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:40:03.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:40:03.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:40:03.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:40:03.735 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:40:03.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:40:03.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:03 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[24757]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:40:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3109789141' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:40:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:40:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[24757]: osdmap e97: 8 total, 8 up, 8 in 2024-05-20T22:40:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[24757]: Standby manager daemon y started 2024-05-20T22:40:04.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[24757]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-20T22:40:04.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[24757]: mgrmap e23: x(active, starting, since 1.00869s), standbys: y 2024-05-20T22:40:04.683 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:04 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:04.361+0000 7f5c0d130700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-20T22:40:04.683 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:04 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ignoring --setuser ceph since I am not root 2024-05-20T22:40:04.683 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:04 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ignoring --setgroup ceph since I am not root 2024-05-20T22:40:04.683 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mgr[24993]: -- 172.21.15.55:0/146467966 <== mon.1 v2:172.21.15.55:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5623faec2340 con 0x5623fae9a800 2024-05-20T22:40:04.683 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:04 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:04.545+0000 7f1960181000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:40:04.683 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:04 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:04.688+0000 7f1960181000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:40:04.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[29498]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:40:04.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3109789141' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:40:04.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:40:04.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[29498]: osdmap e97: 8 total, 8 up, 8 in 2024-05-20T22:40:04.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[29498]: Standby manager daemon y started 2024-05-20T22:40:04.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[29498]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-20T22:40:04.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:04 smithi055 ceph-mon[29498]: mgrmap e23: x(active, starting, since 1.00869s), standbys: y 2024-05-20T22:40:04.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:04 smithi138 ceph-mon[25673]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:40:04.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:04 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3109789141' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:40:04.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:04 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:40:04.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:04 smithi138 ceph-mon[25673]: osdmap e97: 8 total, 8 up, 8 in 2024-05-20T22:40:04.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:04 smithi138 ceph-mon[25673]: Standby manager daemon y started 2024-05-20T22:40:04.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:04 smithi138 ceph-mon[25673]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-20T22:40:04.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:04 smithi138 ceph-mon[25673]: mgrmap e23: x(active, starting, since 1.00869s), standbys: y 2024-05-20T22:40:04.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:04] ENGINE Bus STOPPING 2024-05-20T22:40:04.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T22:40:04.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:04] ENGINE Bus STOPPED 2024-05-20T22:40:04.859 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:04] ENGINE Bus STARTING 2024-05-20T22:40:04.859 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:04] ENGINE Serving on http://:::9283 2024-05-20T22:40:04.859 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:04 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:04] ENGINE Bus STARTED 2024-05-20T22:40:04.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:04 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:04.794+0000 7f1960181000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:40:04.985 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:04 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:40:04.733Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T22:40:04.985 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:04 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:40:04.901Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-05-20T22:40:05.041 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'sleep 180' 2024-05-20T22:40:05.364 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:05 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:05.126+0000 7f1960181000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:40:05.364 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:05 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:05.216+0000 7f1960181000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:40:05.673 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:40:05.673 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:40:05.673 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:40:05.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: Manager daemon x is now available 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: Queued rgw.foo for migration 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.676 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:40:05.676 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:40:05.676 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:40:05.676 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:40:05.676 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:40:05.676 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:40:05.676 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:40:05.676 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: Manager daemon x is now available 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: Queued rgw.foo for migration 2024-05-20T22:40:05.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-20T22:40:05.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:40:05.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:05.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:40:05.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:40:05.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:40:05.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:40:05.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:40:05.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:40:05.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:40:05.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: Manager daemon x is now available 2024-05-20T22:40:05.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: Queued rgw.foo for migration 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:05 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:05.984 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:05 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:40:05.771Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=4 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:40:06.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:40:05] "GET /metrics HTTP/1.1" 200 35099 "" "Prometheus/2.33.4" 2024-05-20T22:40:06.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:06 smithi055 ceph-mon[24757]: Deploying cephadm binary to smithi138 2024-05-20T22:40:06.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:06 smithi055 ceph-mon[24757]: Deploying cephadm binary to smithi055 2024-05-20T22:40:06.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:06 smithi055 ceph-mon[24757]: mgrmap e24: x(active, since 2s), standbys: y 2024-05-20T22:40:06.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:06 smithi055 ceph-mon[24757]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:06.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:06 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:06.444+0000 7f1960181000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:40:06.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:06 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:06.622+0000 7f1960181000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:40:06.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:06 smithi055 ceph-mon[29498]: Deploying cephadm binary to smithi138 2024-05-20T22:40:06.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:06 smithi055 ceph-mon[29498]: Deploying cephadm binary to smithi055 2024-05-20T22:40:06.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:06 smithi055 ceph-mon[29498]: mgrmap e24: x(active, since 2s), standbys: y 2024-05-20T22:40:06.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:06 smithi055 ceph-mon[29498]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:06 smithi138 ceph-mon[25673]: Deploying cephadm binary to smithi138 2024-05-20T22:40:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:06 smithi138 ceph-mon[25673]: Deploying cephadm binary to smithi055 2024-05-20T22:40:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:06 smithi138 ceph-mon[25673]: mgrmap e24: x(active, since 2s), standbys: y 2024-05-20T22:40:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:06 smithi138 ceph-mon[25673]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:06.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:06 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:06.711+0000 7f1960181000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:40:07.402 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:07.039+0000 7f1960181000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:40:07.402 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:07.133+0000 7f1960181000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:40:07.402 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:07.231+0000 7f1960181000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:40:07.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:07.486+0000 7f1960181000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:40:07.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[24757]: [20/May/2024:22:40:05] ENGINE Bus STARTING 2024-05-20T22:40:07.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[24757]: [20/May/2024:22:40:05] ENGINE Serving on https://172.21.15.138:7150 2024-05-20T22:40:07.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[24757]: [20/May/2024:22:40:06] ENGINE Serving on http://172.21.15.138:8765 2024-05-20T22:40:07.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[24757]: [20/May/2024:22:40:06] ENGINE Bus STARTED 2024-05-20T22:40:07.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[24757]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:07.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[29498]: [20/May/2024:22:40:05] ENGINE Bus STARTING 2024-05-20T22:40:07.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[29498]: [20/May/2024:22:40:05] ENGINE Serving on https://172.21.15.138:7150 2024-05-20T22:40:07.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[29498]: [20/May/2024:22:40:06] ENGINE Serving on http://172.21.15.138:8765 2024-05-20T22:40:07.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[29498]: [20/May/2024:22:40:06] ENGINE Bus STARTED 2024-05-20T22:40:07.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:07 smithi055 ceph-mon[29498]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:07 smithi138 ceph-mon[25673]: [20/May/2024:22:40:05] ENGINE Bus STARTING 2024-05-20T22:40:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:07 smithi138 ceph-mon[25673]: [20/May/2024:22:40:05] ENGINE Serving on https://172.21.15.138:7150 2024-05-20T22:40:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:07 smithi138 ceph-mon[25673]: [20/May/2024:22:40:06] ENGINE Serving on http://172.21.15.138:8765 2024-05-20T22:40:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:07 smithi138 ceph-mon[25673]: [20/May/2024:22:40:06] ENGINE Bus STARTED 2024-05-20T22:40:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:07 smithi138 ceph-mon[25673]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:07.954+0000 7f1960181000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:40:08.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:08 smithi055 ceph-mon[24757]: mgrmap e25: x(active, since 4s), standbys: y 2024-05-20T22:40:08.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:08 smithi055 ceph-mon[29498]: mgrmap e25: x(active, since 4s), standbys: y 2024-05-20T22:40:08.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:08 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:08.468+0000 7f1960181000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:40:08.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:08 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:08.567+0000 7f1960181000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:40:08.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:08 smithi138 ceph-mon[25673]: mgrmap e25: x(active, since 4s), standbys: y 2024-05-20T22:40:09.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:08 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:08.876+0000 7f1960181000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:40:09.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:08 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:08.972+0000 7f1960181000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:40:09.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:09 smithi055 ceph-mon[24757]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:09.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:09 smithi055 ceph-mon[29498]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:09.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:09.395+0000 7f1960181000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:40:09.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:09.536+0000 7f1960181000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:40:09.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:09.637+0000 7f1960181000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:40:09.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:09 smithi138 ceph-mon[25673]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:10.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:09.763+0000 7f1960181000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:40:10.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:09.861+0000 7f1960181000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:40:10.682 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:10.471+0000 7f1960181000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:40:10.955 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:10.852+0000 7f1960181000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:40:10.955 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:40:10] ENGINE Bus STARTING 2024-05-20T22:40:10.955 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: CherryPy Checker: 2024-05-20T22:40:10.956 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: The Application mounted at '' has an empty config. 2024-05-20T22:40:10.956 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: 2024-05-20T22:40:11.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:11 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:40:11.020Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:40:11.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:40:10] ENGINE Serving on http://:::9283 2024-05-20T22:40:11.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:10 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: [20/May/2024:22:40:10] ENGINE Bus STARTED 2024-05-20T22:40:11.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[24757]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:11.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[24757]: Standby manager daemon y restarted 2024-05-20T22:40:11.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[24757]: Standby manager daemon y started 2024-05-20T22:40:11.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T22:40:11.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:40:11.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T22:40:11.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:40:11.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[29498]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:11.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[29498]: Standby manager daemon y restarted 2024-05-20T22:40:11.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[29498]: Standby manager daemon y started 2024-05-20T22:40:11.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T22:40:11.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:40:11.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T22:40:11.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:11 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:40:11.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:11 smithi138 ceph-mon[25673]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:11.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:11 smithi138 ceph-mon[25673]: Standby manager daemon y restarted 2024-05-20T22:40:11.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:11 smithi138 ceph-mon[25673]: Standby manager daemon y started 2024-05-20T22:40:11.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:11 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T22:40:11.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:11 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:40:11.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:11 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T22:40:11.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:11 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/2324505656' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:40:12.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:12 smithi055 ceph-mon[24757]: mgrmap e26: x(active, since 8s), standbys: y 2024-05-20T22:40:12.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:12 smithi055 ceph-mon[29498]: mgrmap e26: x(active, since 8s), standbys: y 2024-05-20T22:40:12.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:12 smithi138 ceph-mon[25673]: mgrmap e26: x(active, since 8s), standbys: y 2024-05-20T22:40:13.478 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:13 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:13.478 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:13 smithi138 ceph-mon[25673]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:40:13.478 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:13 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:13.479 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:13 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:13.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:13 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:13.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:13 smithi055 ceph-mon[24757]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:40:13.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:13 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:13.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:13 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:13.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:13 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:13.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:13 smithi055 ceph-mon[29498]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:40:13.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:13 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:13.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:13 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:13.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=error ts=2024-05-20T22:40:13.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:40:13.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:40:13.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.138:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.138 because it doesn't contain any IP SANs" 2024-05-20T22:40:13.736 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=warn ts=2024-05-20T22:40:13.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.55:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.55 because it doesn't contain any IP SANs" 2024-05-20T22:40:15.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi055 to 2993M 2024-05-20T22:40:15.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi055 to 2993M 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:15.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi055 to 2993M 2024-05-20T22:40:15.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:40:15] "GET /metrics HTTP/1.1" 200 35099 "" "Prometheus/2.33.4" 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: Adjusting osd_memory_target on smithi138 to 3248M 2024-05-20T22:40:16.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:40:16.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:15 smithi138 ceph-mon[25673]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:40:16.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: Adjusting osd_memory_target on smithi138 to 3248M 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:40:16.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[24757]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: Adjusting osd_memory_target on smithi138 to 3248M 2024-05-20T22:40:16.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:40:16.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:15 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:40:16.985 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:16 smithi055 systemd[1]: Stopping Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:16.985 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:16 smithi055 bash[58442]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter.a" found: no such container 2024-05-20T22:40:16.985 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:16 smithi055 podman[58453]: 2024-05-20 22:40:16.854463476 +0000 UTC m=+0.137423345 container died 08e55b1e4140078e1bb52d97774f3d3a46ee2c0d80bb253bc28694b958674c63 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:17.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: Reconfiguring daemon node-exporter.a on smithi055 2024-05-20T22:40:17.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:16 smithi138 ceph-mon[25673]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:40:17.371 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:40:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:40:17.371 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 podman[58453]: 2024-05-20 22:40:17.176692488 +0000 UTC m=+0.459652357 container cleanup 08e55b1e4140078e1bb52d97774f3d3a46ee2c0d80bb253bc28694b958674c63 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:17.371 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 bash[58453]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a 2024-05-20T22:40:17.371 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 bash[58478]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter.a" found: no such container 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: Reconfiguring daemon node-exporter.a on smithi055 2024-05-20T22:40:17.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[24757]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: Reconfiguring daemon node-exporter.a on smithi055 2024-05-20T22:40:17.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:16 smithi055 ceph-mon[29498]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:40:17.631 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 podman[58465]: 2024-05-20 22:40:17.378176453 +0000 UTC m=+0.524747196 container remove 08e55b1e4140078e1bb52d97774f3d3a46ee2c0d80bb253bc28694b958674c63 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:17.631 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-20T22:40:17.631 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-20T22:40:17.631 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 systemd[1]: Stopped Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:17.631 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service: Consumed 3.908s CPU time. 2024-05-20T22:40:17.631 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 systemd[1]: Starting Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:17.631 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 podman[58544]: 2024-05-20 22:40:17.52327273 +0000 UTC m=+0.017520954 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-20T22:40:17.631 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 podman[58544]: 2024-05-20 22:40:17.637832252 +0000 UTC m=+0.132080507 container create 69655ac4483b0a3f4a8ab506a852ff288c0da6671921140d5449303cb96d7356 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:17.917 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 podman[58544]: 2024-05-20 22:40:17.840534119 +0000 UTC m=+0.334782343 container init 69655ac4483b0a3f4a8ab506a852ff288c0da6671921140d5449303cb96d7356 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:17.917 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 podman[58544]: 2024-05-20 22:40:17.843917388 +0000 UTC m=+0.338165601 container start 69655ac4483b0a3f4a8ab506a852ff288c0da6671921140d5449303cb96d7356 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.847Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.847Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.847Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.847Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=arp 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-20T22:40:17.918 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-20T22:40:17.919 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-20T22:40:17.919 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-20T22:40:17.919 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=edac 2024-05-20T22:40:17.919 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-20T22:40:17.919 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-20T22:40:17.919 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-20T22:40:17.919 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-20T22:40:17.920 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=os 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=stat 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=time 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-20T22:40:17.921 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=uname 2024-05-20T22:40:17.922 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-20T22:40:17.922 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-20T22:40:17.922 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-20T22:40:17.922 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.848Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-20T22:40:17.922 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[58559]: ts=2024-05-20T22:40:17.849Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-20T22:40:17.922 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 bash[58544]: 69655ac4483b0a3f4a8ab506a852ff288c0da6671921140d5449303cb96d7356 2024-05-20T22:40:18.234 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:40:17 smithi055 systemd[1]: Started Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:19.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:19.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:19.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[24757]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-20T22:40:19.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[24757]: Reconfiguring daemon alertmanager.a on smithi055 2024-05-20T22:40:19.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[24757]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:40:19.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:19.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:19.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[29498]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-20T22:40:19.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[29498]: Reconfiguring daemon alertmanager.a on smithi055 2024-05-20T22:40:19.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:18 smithi055 ceph-mon[29498]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:40:19.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:18 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:19.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:18 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:19.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:18 smithi138 ceph-mon[25673]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-20T22:40:19.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:18 smithi138 ceph-mon[25673]: Reconfiguring daemon alertmanager.a on smithi055 2024-05-20T22:40:19.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:18 smithi138 ceph-mon[25673]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:40:20.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:20.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:21.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:21 smithi055 ceph-mon[24757]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:40:21.135 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:21 smithi055 ceph-mon[29498]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:40:21.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:21 smithi138 ceph-mon[25673]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:40:21.485 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:21 smithi055 systemd[1]: Stopping Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:21.485 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:21 smithi055 bash[58733]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager.a" found: no such container 2024-05-20T22:40:21.485 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:21 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[49371]: level=info ts=2024-05-20T22:40:21.345Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-20T22:40:21.485 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:21 smithi055 podman[58744]: 2024-05-20 22:40:21.366075845 +0000 UTC m=+0.130191086 container died 37600d6e6b2f10d71c806a5f11df501466ff201f46d0eccdfb577a76fc5ca17d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:21.914 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:21 smithi055 podman[58744]: 2024-05-20 22:40:21.745239209 +0000 UTC m=+0.509354449 container cleanup 37600d6e6b2f10d71c806a5f11df501466ff201f46d0eccdfb577a76fc5ca17d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:21.914 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:21 smithi055 bash[58744]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a 2024-05-20T22:40:22.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:21 smithi055 podman[58756]: 2024-05-20 22:40:21.921449208 +0000 UTC m=+0.550542448 container remove 37600d6e6b2f10d71c806a5f11df501466ff201f46d0eccdfb577a76fc5ca17d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:22.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 podman[58756]: 2024-05-20 22:40:22.022276494 +0000 UTC m=+0.651369692 volume remove 6cf369965a506e9f5accc45de02e6f5799be3f5df61b88ad60e0ca0f0ab57f4a 2024-05-20T22:40:22.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 bash[58772]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager.a" found: no such container 2024-05-20T22:40:22.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-20T22:40:22.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 systemd[1]: Stopped Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:22.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@alertmanager.a.service: Consumed 1.051s CPU time. 2024-05-20T22:40:22.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 systemd[1]: Starting Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:22.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:22.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.55:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.138:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:40:22.628 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 podman[58835]: 2024-05-20 22:40:22.261033799 +0000 UTC m=+0.019770217 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-20T22:40:22.628 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 podman[58835]: 2024-05-20 22:40:22.365817393 +0000 UTC m=+0.124553806 volume create 74a641217cebd0b4590dd0a069dbedc1d57bad709e581e758f69f0a1cff11003 2024-05-20T22:40:22.628 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 podman[58835]: 2024-05-20 22:40:22.457735635 +0000 UTC m=+0.216472038 container create a0f540c0ca049b3c4627ff362461ce6088bcde10950e8a3b04a52d15ee6e5878 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:22.985 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 podman[58835]: 2024-05-20 22:40:22.747049826 +0000 UTC m=+0.505786237 container init a0f540c0ca049b3c4627ff362461ce6088bcde10950e8a3b04a52d15ee6e5878 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:22.985 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 podman[58835]: 2024-05-20 22:40:22.750874857 +0000 UTC m=+0.509611276 container start a0f540c0ca049b3c4627ff362461ce6088bcde10950e8a3b04a52d15ee6e5878 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:22.985 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:22.762Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-20T22:40:22.985 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:22.762Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-20T22:40:22.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:22.764Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.55 port=9094 2024-05-20T22:40:22.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:22.765Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-20T22:40:22.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:22.790Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-20T22:40:22.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:22.791Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-20T22:40:22.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:22.793Z caller=main.go:518 msg=Listening address=:9093 2024-05-20T22:40:22.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:22.793Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-20T22:40:22.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 bash[58835]: a0f540c0ca049b3c4627ff362461ce6088bcde10950e8a3b04a52d15ee6e5878 2024-05-20T22:40:22.986 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:22 smithi055 systemd[1]: Started Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:23.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:23 smithi138 ceph-mon[25673]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:23 smithi138 ceph-mon[25673]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:40:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:23 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:23 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:23 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:23 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:23.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:23 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:23.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[24757]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[24757]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[29498]: from='client.14802 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[29498]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:23.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:23.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:23.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:23.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:23 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:24.589 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:24 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:40:24.589 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:24 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:40:24.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:24 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:40:24.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:24 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:40:24.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:24 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:40:24.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:24 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:40:24.857 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:24.766Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000698982s 2024-05-20T22:40:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:25 smithi055 ceph-mon[24757]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:40:25.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:25 smithi055 ceph-mon[29498]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:40:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:25 smithi138 ceph-mon[25673]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:40:26.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:40:25] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:40:25.959+0000 7f48617a5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Creating ceph-iscsi config... 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Traceback (most recent call last): 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return _run_code(code, main_globals, None, 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:40:26.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: exec(code, run_globals) 2024-05-20T22:40:26.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:40:26.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:40:26.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:40:26.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:40:26.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:40:26.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Traceback (most recent call last): 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:40:26.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return future.result(timeout) 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return self.__get_result() 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: raise self._exception 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: use_current_daemon_image=reconfig, 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Creating ceph-iscsi config... 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:40:26.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Traceback (most recent call last): 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return _run_code(code, main_globals, None, 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: exec(code, run_globals) 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:40:26.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:40:26.113 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:26.113 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.485 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:40:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[24989]: ::ffff:172.21.15.138 - - [20/May/2024:22:40:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:40:27.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:27 smithi055 ceph-mon[24757]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:40:27.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:40:27.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:40:27.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:40:27.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: return future.result(timeout) 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.__get_result() 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: raise self._exception 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:40:27.488 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:40:27.489 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:40:27.490 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:40:27.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:27 smithi055 ceph-mon[29498]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:40:27.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:40:27.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:40:27.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:40:27.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:40:27.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: return future.result(timeout) 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.__get_result() 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: raise self._exception 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:40:27.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:40:27.493 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:40:27.494 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:40:27.494 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:40:27.494 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:40:27.494 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:40:27.494 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:40:27.494 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:40:27.494 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:40:27.494 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:27 smithi055 ceph-mon[24757]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:27 smithi055 ceph-mon[24757]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:27 smithi055 ceph-mon[24757]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:27 smithi055 ceph-mon[29498]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:40:27.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:27 smithi055 ceph-mon[29498]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:40:27.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:27 smithi055 ceph-mon[29498]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-20T22:40:27.849 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:27 smithi138 ceph-mon[25673]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:40:27.849 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:40:27.849 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:40:27.850 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:40:27.850 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:40:27.850 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.850 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.850 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:40:27.850 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:40:27.851 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:40:27.851 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:40:27.851 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:40:27.851 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:40:27.851 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:40:27.851 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:40:27.851 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:40:27.851 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:40:27.852 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:40:27.852 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:40:27.852 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.852 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.852 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:40:27.852 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:40:27.853 INFO:journalctl@ceph.mon.b.smithi138.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:40:27.853 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:40:27.853 INFO:journalctl@ceph.mon.b.smithi138.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:40:27.853 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:40:27.853 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:40:27.853 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:40:27.854 INFO:journalctl@ceph.mon.b.smithi138.stdout: return future.result(timeout) 2024-05-20T22:40:27.854 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:40:27.854 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.__get_result() 2024-05-20T22:40:27.854 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:40:27.854 INFO:journalctl@ceph.mon.b.smithi138.stdout: raise self._exception 2024-05-20T22:40:27.854 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:40:27.855 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:40:27.855 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:40:27.855 INFO:journalctl@ceph.mon.b.smithi138.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:40:27.855 INFO:journalctl@ceph.mon.b.smithi138.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:40:27.855 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:40:27.855 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:40:27.855 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:40:27.855 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:40:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:40:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:40:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:40:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:40:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:40:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:40:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:40:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:40:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:40:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:40:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:40:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:40:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:40:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:40:27.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:27 smithi138 ceph-mon[25673]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:40:27.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:27 smithi138 ceph-mon[25673]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:40:27.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:27 smithi138 ceph-mon[25673]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-20T22:40:29.474 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:29 smithi138 ceph-mon[25673]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:40:29.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:29 smithi055 ceph-mon[24757]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:40:29.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:29 smithi055 ceph-mon[29498]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:40:29.771 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 systemd[1]: Stopping Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:29.771 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 bash[48663]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:40:30.054 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.777Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.777Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-20T22:40:30.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[42454]: ts=2024-05-20T22:40:29.778Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-20T22:40:30.056 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:29 smithi138 podman[48674]: 2024-05-20 22:40:29.788011457 +0000 UTC m=+0.115753441 container died 1c5c1e535c2dcc4a869b6ff30c202bc785a9c5d3b8bf641c8af36a1bc925d43d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:30.357 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 podman[48674]: 2024-05-20 22:40:30.152361315 +0000 UTC m=+0.480103299 container cleanup 1c5c1e535c2dcc4a869b6ff30c202bc785a9c5d3b8bf641c8af36a1bc925d43d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:30.357 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 bash[48674]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a 2024-05-20T22:40:30.358 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 bash[48701]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:40:30.682 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 podman[48688]: 2024-05-20 22:40:30.438441528 +0000 UTC m=+0.646237395 container remove 1c5c1e535c2dcc4a869b6ff30c202bc785a9c5d3b8bf641c8af36a1bc925d43d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:30.683 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-20T22:40:30.683 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 systemd[1]: Stopped Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:30.683 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Consumed 1.938s CPU time. 2024-05-20T22:40:30.683 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 systemd[1]: Starting Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:30.683 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 podman[48764]: 2024-05-20 22:40:30.578550241 +0000 UTC m=+0.018298345 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-20T22:40:30.683 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 podman[48764]: 2024-05-20 22:40:30.68964158 +0000 UTC m=+0.129389665 container create f5956ab8bbba15dc4e7c7ad79bd83eb7d86604476b8114b08f35d2aab612f55a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:30.985 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 podman[48764]: 2024-05-20 22:40:30.892658023 +0000 UTC m=+0.332406117 container init f5956ab8bbba15dc4e7c7ad79bd83eb7d86604476b8114b08f35d2aab612f55a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 podman[48764]: 2024-05-20 22:40:30.89555978 +0000 UTC m=+0.335307875 container start f5956ab8bbba15dc4e7c7ad79bd83eb7d86604476b8114b08f35d2aab612f55a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.919Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.919Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.919Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.919Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi138 (none))" 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.919Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.919Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.920Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.921Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.922Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-20T22:40:30.986 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.924Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-20T22:40:30.987 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.924Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.145µs 2024-05-20T22:40:30.987 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:30.924Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-20T22:40:30.987 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 bash[48764]: f5956ab8bbba15dc4e7c7ad79bd83eb7d86604476b8114b08f35d2aab612f55a 2024-05-20T22:40:31.278 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:30 smithi138 systemd[1]: Started Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:31.278 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:31.121Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-05-20T22:40:31.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:31 smithi138 ceph-mon[25673]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:40:31.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:31 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:31.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:31 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:31.579 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:31.388Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-05-20T22:40:31.579 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:31.388Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-05-20T22:40:31.579 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:31.388Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.287µs wal_replay_duration=463.466668ms total_replay_duration=463.510021ms 2024-05-20T22:40:31.580 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:31.393Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-20T22:40:31.580 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:31.393Z caller=main.go:947 level=info msg="TSDB started" 2024-05-20T22:40:31.580 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:31.393Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-20T22:40:31.580 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:31.425Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=31.510355ms db_storage=1.055µs remote_storage=1.854µs web_handler=534ns query_engine=962ns scrape=565.942µs scrape_sd=167.175µs notify=18.477µs notify_sd=16.63µs rules=30.20243ms 2024-05-20T22:40:31.580 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:31.425Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-20T22:40:31.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:31 smithi055 ceph-mon[24757]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:40:31.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:31 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:31.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:31 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:31.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:31 smithi055 ceph-mon[29498]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:40:31.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:31 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:31.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:31 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:31.857 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:31 smithi138 systemd[1]: Stopping Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:31.857 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:31 smithi138 bash[48897]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter.b" found: no such container 2024-05-20T22:40:31.857 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:31 smithi138 podman[48908]: 2024-05-20 22:40:31.723849984 +0000 UTC m=+0.106064192 container died 8a4a20d56038e441c8726617c0418ff5283e9dcae8b99772c4ac6473d5b24b10 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:40:32.551 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:32 smithi138 ceph-mon[25673]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-20T22:40:32.551 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:32 smithi138 ceph-mon[25673]: Reconfiguring daemon node-exporter.b on smithi138 2024-05-20T22:40:32.551 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 podman[48908]: 2024-05-20 22:40:32.305378642 +0000 UTC m=+0.687592841 container cleanup 8a4a20d56038e441c8726617c0418ff5283e9dcae8b99772c4ac6473d5b24b10 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:40:32.551 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 bash[48908]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b 2024-05-20T22:40:32.552 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 bash[48936]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter.b" found: no such container 2024-05-20T22:40:32.552 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 podman[48923]: 2024-05-20 22:40:32.540421472 +0000 UTC m=+0.820126629 container remove 8a4a20d56038e441c8726617c0418ff5283e9dcae8b99772c4ac6473d5b24b10 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:40:32.552 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-20T22:40:32.552 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-20T22:40:32.552 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 systemd[1]: Stopped Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:32.552 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service: Consumed 3.834s CPU time. 2024-05-20T22:40:32.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:32 smithi055 ceph-mon[24757]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-20T22:40:32.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:32 smithi055 ceph-mon[24757]: Reconfiguring daemon node-exporter.b on smithi138 2024-05-20T22:40:32.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:32 smithi055 ceph-mon[29498]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-20T22:40:32.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:32 smithi055 ceph-mon[29498]: Reconfiguring daemon node-exporter.b on smithi138 2024-05-20T22:40:32.857 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 systemd[1]: Starting Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:32.857 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 podman[48999]: 2024-05-20 22:40:32.672502885 +0000 UTC m=+0.016730717 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-20T22:40:32.857 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 podman[48999]: 2024-05-20 22:40:32.792017953 +0000 UTC m=+0.136245783 container create c17852f7fcd918089a90bccd986176be3321cb851273e7cd227db315066c9f1e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:40:33.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:40:32 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:40:32.768Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003266588s 2024-05-20T22:40:33.358 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 podman[48999]: 2024-05-20 22:40:32.994752194 +0000 UTC m=+0.338980027 container init c17852f7fcd918089a90bccd986176be3321cb851273e7cd227db315066c9f1e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:40:33.358 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:32 smithi138 podman[48999]: 2024-05-20 22:40:32.998006302 +0000 UTC m=+0.342234137 container start c17852f7fcd918089a90bccd986176be3321cb851273e7cd227db315066c9f1e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:40:33.358 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.001Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-20T22:40:33.358 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.001Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-20T22:40:33.358 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-20T22:40:33.358 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-20T22:40:33.358 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=arp 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-20T22:40:33.359 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=edac 2024-05-20T22:40:33.360 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-20T22:40:33.360 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-20T22:40:33.360 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-20T22:40:33.360 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-20T22:40:33.361 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=os 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=stat 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-20T22:40:33.362 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=time 2024-05-20T22:40:33.363 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-20T22:40:33.363 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=uname 2024-05-20T22:40:33.363 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-20T22:40:33.363 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-20T22:40:33.363 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.002Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-20T22:40:33.363 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.003Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-20T22:40:33.363 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[49014]: ts=2024-05-20T22:40:33.003Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-20T22:40:33.363 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 bash[48999]: c17852f7fcd918089a90bccd986176be3321cb851273e7cd227db315066c9f1e 2024-05-20T22:40:33.363 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:40:33 smithi138 systemd[1]: Started Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[24757]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:40:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:40:33.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[29498]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:40:33.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:33 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:40:33.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-mon[25673]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:40:33.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:33.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:33 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:40:34.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:34 smithi138 ceph-mon[25673]: Reconfiguring grafana.a (dependencies changed)... 2024-05-20T22:40:34.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:34 smithi138 ceph-mon[25673]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-20T22:40:34.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:34 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:40:34.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:34 smithi138 ceph-mon[25673]: Reconfiguring daemon grafana.a on smithi138 2024-05-20T22:40:34.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:34 smithi055 ceph-mon[24757]: Reconfiguring grafana.a (dependencies changed)... 2024-05-20T22:40:34.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:34 smithi055 ceph-mon[24757]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-20T22:40:34.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:34 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:40:34.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:34 smithi055 ceph-mon[24757]: Reconfiguring daemon grafana.a on smithi138 2024-05-20T22:40:34.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:34 smithi055 ceph-mon[29498]: Reconfiguring grafana.a (dependencies changed)... 2024-05-20T22:40:34.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:34 smithi055 ceph-mon[29498]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-20T22:40:34.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:34 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:40:34.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:34 smithi055 ceph-mon[29498]: Reconfiguring daemon grafana.a on smithi138 2024-05-20T22:40:35.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:35 smithi055 ceph-mon[24757]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:40:35.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:35 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:35.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:35 smithi055 ceph-mon[29498]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:40:35.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:35 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:35.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:35 smithi138 ceph-mon[25673]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:40:35.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:35 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:36.357 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 systemd[1]: Stopping Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:36.357 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 bash[49186]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana.a" found: no such container 2024-05-20T22:40:36.357 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[41684]: t=2024-05-20T22:40:36+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-05-20T22:40:36.358 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 podman[49198]: 2024-05-20 22:40:36.164760355 +0000 UTC m=+0.119872245 container died 93d02684b943a111c3796a5b2362ee0edf510b2f99ba720844178f2d0cdaf198 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, vcs-type=git, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, release=236.1648460182, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vendor=Red Hat, Inc., io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8) 2024-05-20T22:40:36.691 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 podman[49198]: 2024-05-20 22:40:36.538319542 +0000 UTC m=+0.493431432 container cleanup 93d02684b943a111c3796a5b2362ee0edf510b2f99ba720844178f2d0cdaf198 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, vcs-type=git, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, version=8.5) 2024-05-20T22:40:36.691 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 bash[49198]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a 2024-05-20T22:40:36.964 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 podman[49210]: 2024-05-20 22:40:36.698120272 +0000 UTC m=+0.527774447 container remove 93d02684b943a111c3796a5b2362ee0edf510b2f99ba720844178f2d0cdaf198 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.buildah.version=1.24.2, maintainer=Paul Cuzner , version=8.5, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.openshift.expose-services=, vcs-type=git, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-05-20T22:40:36.964 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 bash[49225]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana.a" found: no such container 2024-05-20T22:40:36.964 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-20T22:40:36.964 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 systemd[1]: Stopped Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:36.964 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@grafana.a.service: Consumed 3.259s CPU time. 2024-05-20T22:40:36.964 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 systemd[1]: Starting Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:40:37.237 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:36 smithi138 podman[49286]: 2024-05-20 22:40:36.872808983 +0000 UTC m=+0.016545071 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-05-20T22:40:37.237 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 podman[49286]: 2024-05-20 22:40:37.007717755 +0000 UTC m=+0.151453842 container create 179b97bd74004e079a70c30c88db3819f289c15c4147555f756ab768dec8f7ca (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, description=Ceph Grafana Container, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.openshift.tags=base rhel8, release=236.1648460182, architecture=x86_64, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=) 2024-05-20T22:40:37.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:37 smithi138 ceph-mon[25673]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:40:37.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:37 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:37.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:37 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:37.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:37 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:40:37.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:37 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:40:37.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:37 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:37.584 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STOPPING 2024-05-20T22:40:37.584 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T22:40:37.584 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STOPPED 2024-05-20T22:40:37.584 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STARTING 2024-05-20T22:40:37.584 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Serving on http://:::9283 2024-05-20T22:40:37.585 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STARTED 2024-05-20T22:40:37.585 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 podman[49286]: 2024-05-20 22:40:37.244495734 +0000 UTC m=+0.388231823 container init 179b97bd74004e079a70c30c88db3819f289c15c4147555f756ab768dec8f7ca (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, vendor=Red Hat, Inc., release=236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.24.2, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, version=8.5, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container) 2024-05-20T22:40:37.585 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 podman[49286]: 2024-05-20 22:40:37.247710951 +0000 UTC m=+0.391447042 container start 179b97bd74004e079a70c30c88db3819f289c15c4147555f756ab768dec8f7ca (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.buildah.version=1.24.2, distribution-scope=public, vcs-type=git, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, description=Ceph Grafana Container, vendor=Red Hat, Inc., name=ubi8) 2024-05-20T22:40:37.585 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-05-20T22:40:37.585 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-05-20T22:40:37.585 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-05-20T22:40:37.585 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="App mode production" logger=settings 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-05-20T22:40:37.586 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=506.614µs 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 bash[49286]: 179b97bd74004e079a70c30c88db3819f289c15c4147555f756ab768dec8f7ca 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 systemd[1]: Started Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-20T22:40:37.587 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-05-20T22:40:37.588 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-05-20T22:40:37.588 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-05-20T22:40:37.588 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:40:37+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-05-20T22:40:37.610 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[24757]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:40:37.610 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:37.610 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:37.610 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:40:37.610 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:40:37.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:37.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[29498]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:40:37.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:37.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:37.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:40:37.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:40:37.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:37 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:37.857 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STOPPING 2024-05-20T22:40:37.857 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T22:40:37.857 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STOPPED 2024-05-20T22:40:37.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STARTING 2024-05-20T22:40:37.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Serving on http://:::9283 2024-05-20T22:40:37.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STARTED 2024-05-20T22:40:37.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STOPPING 2024-05-20T22:40:37.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T22:40:37.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STOPPED 2024-05-20T22:40:37.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STARTING 2024-05-20T22:40:37.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Serving on http://:::9283 2024-05-20T22:40:37.858 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:40:37] ENGINE Bus STARTED 2024-05-20T22:40:38.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:38.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:40:38.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3834612781' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:40:38.609 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:38 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1865110959' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/53762980"}]: dispatch 2024-05-20T22:40:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:40:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:40:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:40:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:40:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:40:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:40:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:40:38.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3834612781' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1865110959' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/53762980"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:40:38.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:40:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:40:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:40:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:40:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:40:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:40:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3834612781' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:40:38.737 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:38 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1865110959' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/53762980"}]: dispatch 2024-05-20T22:40:39.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[29498]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:39.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1865110959' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/53762980"}]': finished 2024-05-20T22:40:39.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[29498]: osdmap e98: 8 total, 8 up, 8 in 2024-05-20T22:40:39.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2394796782' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2831507036"}]: dispatch 2024-05-20T22:40:39.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2831507036"}]: dispatch 2024-05-20T22:40:39.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[24757]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:39.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1865110959' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/53762980"}]': finished 2024-05-20T22:40:39.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[24757]: osdmap e98: 8 total, 8 up, 8 in 2024-05-20T22:40:39.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2394796782' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2831507036"}]: dispatch 2024-05-20T22:40:39.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:39 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2831507036"}]: dispatch 2024-05-20T22:40:39.841 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:39 smithi138 ceph-mon[25673]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:39.841 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:39 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1865110959' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/53762980"}]': finished 2024-05-20T22:40:39.841 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:39 smithi138 ceph-mon[25673]: osdmap e98: 8 total, 8 up, 8 in 2024-05-20T22:40:39.841 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:39 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2394796782' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2831507036"}]: dispatch 2024-05-20T22:40:39.841 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:39 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2831507036"}]: dispatch 2024-05-20T22:40:40.831 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:40 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2831507036"}]': finished 2024-05-20T22:40:40.832 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:40 smithi138 ceph-mon[25673]: osdmap e99: 8 total, 8 up, 8 in 2024-05-20T22:40:40.832 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:40 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/438301044' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1780260259"}]: dispatch 2024-05-20T22:40:40.875 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:40 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2831507036"}]': finished 2024-05-20T22:40:40.876 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:40 smithi055 ceph-mon[24757]: osdmap e99: 8 total, 8 up, 8 in 2024-05-20T22:40:40.876 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:40 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/438301044' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1780260259"}]: dispatch 2024-05-20T22:40:40.876 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:40 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2831507036"}]': finished 2024-05-20T22:40:40.876 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:40 smithi055 ceph-mon[29498]: osdmap e99: 8 total, 8 up, 8 in 2024-05-20T22:40:40.876 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:40 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/438301044' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1780260259"}]: dispatch 2024-05-20T22:40:41.494 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:40:41] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-20T22:40:41.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:41 smithi138 ceph-mon[25673]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:41.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:41 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/438301044' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1780260259"}]': finished 2024-05-20T22:40:41.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:41 smithi138 ceph-mon[25673]: osdmap e100: 8 total, 8 up, 8 in 2024-05-20T22:40:41.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:41 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2174259878' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2503496786"}]: dispatch 2024-05-20T22:40:41.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:41 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2503496786"}]: dispatch 2024-05-20T22:40:41.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:41 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:41.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:41 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:41.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[29498]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:41.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/438301044' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1780260259"}]': finished 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[29498]: osdmap e100: 8 total, 8 up, 8 in 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2174259878' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2503496786"}]: dispatch 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2503496786"}]: dispatch 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[24757]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/438301044' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/1780260259"}]': finished 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[24757]: osdmap e100: 8 total, 8 up, 8 in 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2174259878' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2503496786"}]: dispatch 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2503496786"}]: dispatch 2024-05-20T22:40:41.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:41.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:41 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2503496786"}]': finished 2024-05-20T22:40:42.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: osdmap e101: 8 total, 8 up, 8 in 2024-05-20T22:40:42.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:40:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2691984439' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3046159667"}]: dispatch 2024-05-20T22:40:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:42.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2503496786"}]': finished 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: osdmap e101: 8 total, 8 up, 8 in 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2691984439' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3046159667"}]: dispatch 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:42 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:42.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2503496786"}]': finished 2024-05-20T22:40:42.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: osdmap e101: 8 total, 8 up, 8 in 2024-05-20T22:40:42.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:42.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:40:42.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:42.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2691984439' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3046159667"}]: dispatch 2024-05-20T22:40:42.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:42.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:40:42.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:42 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:40:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:40:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[24757]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:40:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2691984439' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3046159667"}]': finished 2024-05-20T22:40:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[24757]: osdmap e102: 8 total, 8 up, 8 in 2024-05-20T22:40:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/705192866' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/159719434"}]: dispatch 2024-05-20T22:40:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/159719434"}]: dispatch 2024-05-20T22:40:43.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:40:43.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:40:43.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[29498]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:40:43.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2691984439' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3046159667"}]': finished 2024-05-20T22:40:43.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[29498]: osdmap e102: 8 total, 8 up, 8 in 2024-05-20T22:40:43.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/705192866' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/159719434"}]: dispatch 2024-05-20T22:40:43.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:43 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/159719434"}]: dispatch 2024-05-20T22:40:43.759 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:40:43.760 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:43 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:40:43.760 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:43 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:40:43.760 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:43 smithi138 ceph-mon[25673]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:40:43.760 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:43 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2691984439' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3046159667"}]': finished 2024-05-20T22:40:43.760 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:43 smithi138 ceph-mon[25673]: osdmap e102: 8 total, 8 up, 8 in 2024-05-20T22:40:43.760 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:43 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/705192866' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/159719434"}]: dispatch 2024-05-20T22:40:43.760 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:43 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/159719434"}]: dispatch 2024-05-20T22:40:45.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:44 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/159719434"}]': finished 2024-05-20T22:40:45.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:44 smithi138 ceph-mon[25673]: osdmap e103: 8 total, 8 up, 8 in 2024-05-20T22:40:45.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:44 smithi138 ceph-mon[25673]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:40:45.154 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:44 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/159719434"}]': finished 2024-05-20T22:40:45.154 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:44 smithi055 ceph-mon[24757]: osdmap e103: 8 total, 8 up, 8 in 2024-05-20T22:40:45.154 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:44 smithi055 ceph-mon[24757]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:40:45.155 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:44 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/159719434"}]': finished 2024-05-20T22:40:45.155 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:44 smithi055 ceph-mon[29498]: osdmap e103: 8 total, 8 up, 8 in 2024-05-20T22:40:45.155 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:44 smithi055 ceph-mon[29498]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:40:47.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:47.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:47.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:40:47.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:40:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:40:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:40:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:40:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 699 B/s rd, 0 op/s 2024-05-20T22:40:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1665185230' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:40:47.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2195661748' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/159719434"}]: dispatch 2024-05-20T22:40:47.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:46 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/159719434"}]: dispatch 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 699 B/s rd, 0 op/s 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1665185230' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2195661748' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/159719434"}]: dispatch 2024-05-20T22:40:47.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/159719434"}]: dispatch 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 699 B/s rd, 0 op/s 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1665185230' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2195661748' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/159719434"}]: dispatch 2024-05-20T22:40:47.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:46 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/159719434"}]: dispatch 2024-05-20T22:40:48.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:47 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/159719434"}]': finished 2024-05-20T22:40:48.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:47 smithi055 ceph-mon[24757]: osdmap e104: 8 total, 8 up, 8 in 2024-05-20T22:40:48.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:47 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/159719434"}]': finished 2024-05-20T22:40:48.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:47 smithi055 ceph-mon[29498]: osdmap e104: 8 total, 8 up, 8 in 2024-05-20T22:40:48.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:47 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/159719434"}]': finished 2024-05-20T22:40:48.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:47 smithi138 ceph-mon[25673]: osdmap e104: 8 total, 8 up, 8 in 2024-05-20T22:40:48.857 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:48 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:40:49.277 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:49 smithi055 ceph-mon[24757]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T22:40:49.277 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:49 smithi055 ceph-mon[29498]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T22:40:49.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:49 smithi138 ceph-mon[25673]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T22:40:50.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:50 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:50.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:50 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:50.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:50 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:50.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:50 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:50.828 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:50 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:50.828 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:50 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:50.829 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:50 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:40:50.829 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:50 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:50.829 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:50 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.607 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:40:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:40:51] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-05-20T22:40:51.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:51 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:51 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:51 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:51.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:51 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:40:51.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:51 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:51 smithi138 ceph-mon[25673]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 541 B/s rd, 0 op/s 2024-05-20T22:40:51.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:51.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:40:51.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[29498]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 541 B/s rd, 0 op/s 2024-05-20T22:40:51.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:40:51.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:40:51.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:40:51.736 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:51 smithi055 ceph-mon[24757]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 541 B/s rd, 0 op/s 2024-05-20T22:40:53.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:53 smithi138 ceph-mon[25673]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T22:40:53.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:40:53.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:53 smithi055 ceph-mon[24757]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T22:40:53.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:53 smithi055 ceph-mon[29498]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T22:40:55.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:55 smithi055 ceph-mon[24757]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 1 op/s 2024-05-20T22:40:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:55 smithi055 ceph-mon[29498]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 1 op/s 2024-05-20T22:40:55.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:55 smithi138 ceph-mon[25673]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 1 op/s 2024-05-20T22:40:57.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:57 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:57.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:57 smithi055 ceph-mon[24757]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:40:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:57 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:57 smithi055 ceph-mon[29498]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:40:57.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:57 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:40:57.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:57 smithi138 ceph-mon[25673]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:40:59.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:40:58 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:40:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:40:59.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:40:59 smithi055 ceph-mon[24757]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-05-20T22:40:59.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:40:59 smithi055 ceph-mon[29498]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-05-20T22:40:59.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:40:59 smithi138 ceph-mon[25673]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-05-20T22:41:01.483 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:41:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:41:01] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-05-20T22:41:01.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:01 smithi055 ceph-mon[24757]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:01.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:01 smithi055 ceph-mon[29498]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:01.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:01 smithi138 ceph-mon[25673]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:03.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:03 smithi138 ceph-mon[25673]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:03.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:03 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:03.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:03 smithi055 ceph-mon[24757]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:03.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:03 smithi055 ceph-mon[29498]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:05.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:05 smithi055 ceph-mon[24757]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:05.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:05.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:05 smithi055 ceph-mon[29498]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:05.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:05 smithi138 ceph-mon[25673]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:07.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:07 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:07.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:07 smithi055 ceph-mon[24757]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:07.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:07 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:07.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:07 smithi055 ceph-mon[29498]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:07 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:07 smithi138 ceph-mon[25673]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:09.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:09.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:09 smithi055 ceph-mon[24757]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:09.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:09 smithi055 ceph-mon[29498]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:09.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:09 smithi138 ceph-mon[25673]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:11.462 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:41:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:41:11] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-05-20T22:41:11.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:11 smithi055 ceph-mon[24757]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:11.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:11 smithi055 ceph-mon[29498]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:11.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:11 smithi138 ceph-mon[25673]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:13.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:13 smithi138 ceph-mon[25673]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:13.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:13.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:13 smithi055 ceph-mon[24757]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:13.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:13 smithi055 ceph-mon[29498]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:15.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:15 smithi055 ceph-mon[24757]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:15.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:15 smithi055 ceph-mon[29498]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:15.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:15 smithi138 ceph-mon[25673]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:17 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:17.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:17 smithi055 ceph-mon[24757]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:17 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:17 smithi055 ceph-mon[29498]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:17.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:17 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:17.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:17 smithi138 ceph-mon[25673]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:19.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:19.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:19 smithi055 ceph-mon[24757]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:19.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:19 smithi055 ceph-mon[29498]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:19.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:19 smithi138 ceph-mon[25673]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:20 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:20.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:20 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:20.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:20 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:21.478 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:41:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:41:21] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-05-20T22:41:21.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:21 smithi055 ceph-mon[24757]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:21.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:21 smithi055 ceph-mon[29498]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:21.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:21 smithi138 ceph-mon[25673]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:23.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:23 smithi138 ceph-mon[25673]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:23.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:23.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:23 smithi055 ceph-mon[29498]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:23.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:23 smithi055 ceph-mon[24757]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:25.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:25 smithi055 ceph-mon[24757]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:25.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:25 smithi055 ceph-mon[29498]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:25.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:25 smithi138 ceph-mon[25673]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:27.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:27 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:27.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:27 smithi055 ceph-mon[24757]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:27.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:27 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:27.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:27 smithi055 ceph-mon[29498]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:27 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:27 smithi138 ceph-mon[25673]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:29.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:29.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:29 smithi055 ceph-mon[24757]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:29.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:29 smithi055 ceph-mon[29498]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:29.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:29 smithi138 ceph-mon[25673]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:31.497 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:41:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:41:31] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-05-20T22:41:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:31 smithi138 ceph-mon[25673]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:31.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:31 smithi055 ceph-mon[24757]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:31 smithi055 ceph-mon[29498]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:33 smithi138 ceph-mon[25673]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:33.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:33.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:33 smithi055 ceph-mon[24757]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:33.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:33 smithi055 ceph-mon[29498]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:35.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:35 smithi055 ceph-mon[24757]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:35.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:35 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:35.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:35 smithi055 ceph-mon[29498]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:35.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:35 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:35.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:35 smithi138 ceph-mon[25673]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:35.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:35 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:37.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:37 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:37.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:37 smithi055 ceph-mon[24757]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:37.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:37 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:37.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:37 smithi055 ceph-mon[29498]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:37 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:37 smithi138 ceph-mon[25673]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:39.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:39.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:39 smithi055 ceph-mon[24757]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:39.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:39 smithi055 ceph-mon[29498]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:39.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:39 smithi138 ceph-mon[25673]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:41.473 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:41:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:41:41] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-05-20T22:41:41.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:41 smithi055 ceph-mon[24757]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:41.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:41 smithi055 ceph-mon[29498]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:41.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:41 smithi138 ceph-mon[25673]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:43.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:43 smithi138 ceph-mon[25673]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:43.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:43 smithi055 ceph-mon[24757]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:43.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:43 smithi055 ceph-mon[29498]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:45.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:45 smithi055 ceph-mon[24757]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:45.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:45 smithi055 ceph-mon[29498]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:45.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:45 smithi138 ceph-mon[25673]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:47.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:47 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:47.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:47 smithi055 ceph-mon[24757]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:47.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:47 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:47.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:47 smithi055 ceph-mon[29498]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:47.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:47 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:47.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:47 smithi138 ceph-mon[25673]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:49.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:48 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:49.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:49 smithi055 ceph-mon[24757]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:49.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:49 smithi055 ceph-mon[29498]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:49.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:49 smithi138 ceph-mon[25673]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:50.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:50 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:50.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:50 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:41:50.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:50 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:50.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:50 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:41:50.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:50 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:41:50.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:50 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:41:51.482 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:41:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:41:51] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-20T22:41:51.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:51 smithi055 ceph-mon[24757]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:51.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:51 smithi055 ceph-mon[29498]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:51.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:51 smithi138 ceph-mon[25673]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:53.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:53 smithi055 ceph-mon[24757]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:53.584 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:53 smithi055 ceph-mon[29498]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:53.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:53 smithi138 ceph-mon[25673]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:53.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:55.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:55 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:55 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:55 smithi138 ceph-mon[25673]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:55.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:55 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:55 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[24757]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:55.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[29498]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:55 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:56.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:56 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:41:56.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:56 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:41:56.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:56 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:56.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:56 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:41:56.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:56 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:41:56.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:56 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:56.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:56 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:41:56.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:56 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:41:56.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:56 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:41:57.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:57 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:57.608 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:57 smithi138 ceph-mon[25673]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:57 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:57 smithi055 ceph-mon[24757]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:57 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:41:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:57 smithi055 ceph-mon[29498]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:41:59.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:41:58 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:41:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:41:59.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:41:59 smithi055 ceph-mon[24757]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:59.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:41:59 smithi055 ceph-mon[29498]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:41:59.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:41:59 smithi138 ceph-mon[25673]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:01.484 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:42:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:42:01] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-20T22:42:01.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:01 smithi055 ceph-mon[24757]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:01.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:01 smithi055 ceph-mon[29498]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:01.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:01 smithi138 ceph-mon[25673]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:03.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:03 smithi138 ceph-mon[25673]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:03.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:03 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:03.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:03 smithi055 ceph-mon[24757]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:03.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:03 smithi055 ceph-mon[29498]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:05.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:05 smithi055 ceph-mon[24757]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:05.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:05.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:05 smithi055 ceph-mon[29498]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:05.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:05 smithi138 ceph-mon[25673]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:07.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:07 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:07.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:07 smithi055 ceph-mon[24757]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:07.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:07 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:07.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:07 smithi055 ceph-mon[29498]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:07 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:07 smithi138 ceph-mon[25673]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:09.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:09.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:09 smithi055 ceph-mon[24757]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:09.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:09 smithi055 ceph-mon[29498]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:09.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:09 smithi138 ceph-mon[25673]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:11.474 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:42:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:42:11] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-05-20T22:42:11.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:11 smithi055 ceph-mon[24757]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:11.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:11 smithi055 ceph-mon[29498]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:11.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:11 smithi138 ceph-mon[25673]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:13.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:13 smithi138 ceph-mon[25673]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:13.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:13.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:13 smithi055 ceph-mon[24757]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:13.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:13 smithi055 ceph-mon[29498]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:15.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:15 smithi055 ceph-mon[24757]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:15.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:15 smithi055 ceph-mon[29498]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:15.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:15 smithi138 ceph-mon[25673]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:17 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:17.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:17 smithi055 ceph-mon[24757]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:17 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:17 smithi055 ceph-mon[29498]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:17.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:17 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:17.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:17 smithi138 ceph-mon[25673]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:19.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:19.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:19 smithi055 ceph-mon[24757]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:19.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:19.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:19 smithi055 ceph-mon[29498]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:19.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:19.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:19 smithi138 ceph-mon[25673]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:19.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:21.495 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:42:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:42:21] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-20T22:42:21.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:21 smithi138 ceph-mon[25673]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:21.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:21 smithi055 ceph-mon[24757]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:21.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:21 smithi055 ceph-mon[29498]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:23.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:23 smithi138 ceph-mon[25673]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:23.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:23.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:23 smithi055 ceph-mon[24757]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:23.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:23 smithi055 ceph-mon[29498]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:25.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:25 smithi055 ceph-mon[24757]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:25.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:25 smithi055 ceph-mon[29498]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:25.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:25 smithi138 ceph-mon[25673]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:27.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:27 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:27.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:27 smithi055 ceph-mon[24757]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:27.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:27 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:27.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:27 smithi055 ceph-mon[29498]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:27 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:27 smithi138 ceph-mon[25673]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:29.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:29.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:29 smithi138 ceph-mon[25673]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:29.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:29 smithi055 ceph-mon[24757]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:29.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:29 smithi055 ceph-mon[29498]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:31.505 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:42:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:42:31] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-20T22:42:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:31 smithi138 ceph-mon[25673]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:31.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:31 smithi055 ceph-mon[24757]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:31.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:31 smithi055 ceph-mon[29498]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:33 smithi138 ceph-mon[25673]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:33.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:33.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:33 smithi055 ceph-mon[24757]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:33.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:33 smithi055 ceph-mon[29498]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:35.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:35 smithi055 ceph-mon[24757]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:35.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:35 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:35.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:35 smithi055 ceph-mon[29498]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:35.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:35 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:35.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:35 smithi138 ceph-mon[25673]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:35.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:35 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:37.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:37 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:37.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:37 smithi055 ceph-mon[24757]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:37.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:37 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:37.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:37 smithi055 ceph-mon[29498]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:37 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:37 smithi138 ceph-mon[25673]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:39.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:39.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:39 smithi055 ceph-mon[24757]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:39.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:39 smithi055 ceph-mon[29498]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:39.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:39 smithi138 ceph-mon[25673]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:41.489 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:42:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:42:41] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-20T22:42:41.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:41 smithi138 ceph-mon[25673]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:41.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:41 smithi055 ceph-mon[24757]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:41.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:41 smithi055 ceph-mon[29498]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:43.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:43 smithi138 ceph-mon[25673]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:43.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:43 smithi055 ceph-mon[24757]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:43.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:43 smithi055 ceph-mon[29498]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:45.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:45 smithi055 ceph-mon[24757]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:45.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:45 smithi055 ceph-mon[29498]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:45.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:45 smithi138 ceph-mon[25673]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:47.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:47 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:47.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:47 smithi055 ceph-mon[29498]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:47.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:47 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:47.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:47 smithi055 ceph-mon[24757]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:47.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:47 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:47.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:47 smithi138 ceph-mon[25673]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:49.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:48 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:49.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:49 smithi055 ceph-mon[24757]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:49.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:49 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:49.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:49 smithi055 ceph-mon[29498]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:49.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:49 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:49.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:49 smithi138 ceph-mon[25673]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:49.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:49 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:42:51.500 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:42:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:42:51] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-05-20T22:42:51.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:51 smithi138 ceph-mon[25673]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:51.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:51 smithi055 ceph-mon[24757]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:51.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:51 smithi055 ceph-mon[29498]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:53.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:53 smithi138 ceph-mon[25673]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:53.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:53.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:53 smithi055 ceph-mon[24757]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:53.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:53 smithi055 ceph-mon[29498]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:55.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:55 smithi055 ceph-mon[24757]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:55.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:55 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:42:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:55 smithi055 ceph-mon[29498]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:55 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:42:55.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:55 smithi138 ceph-mon[25673]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:55.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:55 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:42:56.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:56 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:42:56.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:56 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:42:56.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:56 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:42:56.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:56 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:42:56.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:56 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:42:56.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:56 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:42:56.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:56 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:42:56.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:56 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:42:56.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:56 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:42:57.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:57 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:57.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:57 smithi138 ceph-mon[25673]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:57 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:57 smithi055 ceph-mon[24757]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:57 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:42:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:57 smithi055 ceph-mon[29498]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:42:59.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:42:58 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:42:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:42:59.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:42:59 smithi138 ceph-mon[25673]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:42:59 smithi055 ceph-mon[24757]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:42:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:42:59 smithi055 ceph-mon[29498]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:01.515 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:43:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:43:01] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-05-20T22:43:01.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:01 smithi138 ceph-mon[25673]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:01.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:01 smithi055 ceph-mon[24757]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:01.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:01 smithi055 ceph-mon[29498]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:03.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:03 smithi138 ceph-mon[25673]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:03.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:03 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:03.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:03 smithi055 ceph-mon[24757]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:03.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:03 smithi055 ceph-mon[29498]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:05.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:05 smithi055 ceph-mon[24757]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:05.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:05 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:05.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:05 smithi055 ceph-mon[29498]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:05.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:05 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:05 smithi138 ceph-mon[25673]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:05 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:07.520 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-20T22:43:07.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:07 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:07.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:07 smithi055 ceph-mon[24757]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:07.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:07 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:07.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:07 smithi055 ceph-mon[29498]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:07 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:07 smithi138 ceph-mon[25673]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:09.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:09.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:09 smithi138 ceph-mon[25673]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:09.916 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled to redeploy mgr.y on host 'smithi055' 2024-05-20T22:43:09.916 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:09 smithi055 ceph-mon[24757]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:09.917 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:09 smithi055 ceph-mon[29498]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:10.540 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps --refresh' 2024-05-20T22:43:10.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/4033144196' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:43:10.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:10.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:10.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:10.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:43:10.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4033144196' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:43:10.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:10.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:10.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:10.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:10 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:43:10.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:10 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/4033144196' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:43:10.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:10 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:10.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:10 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:10.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:10 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:10.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:10 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:43:11.493 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:43:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:43:11] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-20T22:43:11.808 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:11 smithi055 ceph-mon[29498]: from='client.25066 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:43:11.808 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:11 smithi055 ceph-mon[29498]: Schedule redeploy daemon mgr.y 2024-05-20T22:43:11.808 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:11 smithi055 ceph-mon[29498]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:11.808 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:11 smithi055 ceph-mon[24757]: from='client.25066 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:43:11.808 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:11 smithi055 ceph-mon[24757]: Schedule redeploy daemon mgr.y 2024-05-20T22:43:11.808 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:11 smithi055 ceph-mon[24757]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:11.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:11 smithi138 ceph-mon[25673]: from='client.25066 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:43:11.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:11 smithi138 ceph-mon[25673]: Schedule redeploy daemon mgr.y 2024-05-20T22:43:11.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:11 smithi138 ceph-mon[25673]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (2m) 78s ago 10m 16.4M - 0.23.0 ba2b418f427c a0f540c0ca04 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (2m) 78s ago 10m 41.3M - 8.3.5 dad864ee21e9 179b97bd7400 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (2m) 78s ago 9m 46.3M - 3.5 e1d6a67b021e 0f3d86f87e62 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283 running (5m) 78s ago 15m 512M - 19.0.0-2695-g8a373da3 86045c3f4e75 0639e81ef3c8 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:9283 running (16m) 78s ago 16m 413M - 17.2.0 e1d6a67b021e 70d348a53fb9 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (16m) 78s ago 16m 57.6M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (15m) 78s ago 15m 48.3M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (15m) 78s ago 15m 48.1M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (2m) 78s ago 11m 18.2M - 1.3.1 1dbe0e931976 69655ac4483b 2024-05-20T22:43:12.701 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (2m) 78s ago 11m 8308k - 1.3.1 1dbe0e931976 c17852f7fcd9 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (14m) 78s ago 14m 53.0M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (14m) 78s ago 14m 52.5M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (14m) 78s ago 14m 50.3M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (13m) 78s ago 13m 51.2M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (13m) 78s ago 13m 51.2M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (12m) 78s ago 12m 49.6M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (12m) 78s ago 12m 48.4M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (12m) 78s ago 12m 50.5M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (2m) 78s ago 11m 62.2M - 2.33.4 514e6a882f6e f5956ab8bbba 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (9m) 78s ago 9m 71.3M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:43:12.702 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (9m) 78s ago 9m 74.6M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:43:13.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:13 smithi055 ceph-mon[24757]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:13.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:13 smithi055 ceph-mon[24757]: from='client.25072 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:43:13.494 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:13 smithi055 ceph-mon[29498]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:13.494 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:13 smithi055 ceph-mon[29498]: from='client.25072 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:43:13.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:13 smithi138 ceph-mon[25673]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:13.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:13 smithi138 ceph-mon[25673]: from='client.25072 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:43:13.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:14.113 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'sleep 180' 2024-05-20T22:43:14.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:14 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:14.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:14 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:14.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:14 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:15.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:15 smithi055 ceph-mon[24757]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:15.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:15 smithi055 ceph-mon[29498]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:15.839 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:15 smithi138 ceph-mon[25673]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:17 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:17 smithi055 ceph-mon[24757]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:17.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:17 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:17 smithi055 ceph-mon[29498]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:17.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:17 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:17.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:17 smithi138 ceph-mon[25673]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:18.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:18 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:18.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:18 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:18.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:18 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:18.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:18 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:19.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:18 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:19.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:18 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:19.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:20.103 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:20.103 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:20.103 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:20.103 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:43:20.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:43:20.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:20.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:43:20.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:43:20.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:43:20.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:43:20.105 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:20.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:20.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:20.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:20.106 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:43:20.106 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:43:20.106 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:20.106 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:43:20.106 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:43:20.106 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:43:20.107 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:43:20.107 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:20.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:20.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:20.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:20.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:43:20.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:43:20.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:20.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:43:20.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:43:20.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:43:20.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:43:20.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:20.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:20 smithi055 ceph-mon[24757]: Deploying daemon mgr.y on smithi055 2024-05-20T22:43:20.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:20 smithi055 systemd[1]: Stopping Ceph mgr.y for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:43:20.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:20 smithi055 ceph-mon[29498]: Deploying daemon mgr.y on smithi055 2024-05-20T22:43:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:20 smithi138 ceph-mon[25673]: Deploying daemon mgr.y on smithi055 2024-05-20T22:43:21.283 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:21 smithi055 podman[62937]: 2024-05-20 22:43:21.029139633 +0000 UTC m=+0.157869485 container died 70d348a53fb955a3abed20c7ca388d7651966e06565a4e0b84ee14c02c9cb8c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., distribution-scope=public, release=754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-20T22:43:21.534 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:21 smithi055 podman[62937]: 2024-05-20 22:43:21.37291703 +0000 UTC m=+0.501646882 container cleanup 70d348a53fb955a3abed20c7ca388d7651966e06565a4e0b84ee14c02c9cb8c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, version=8, vcs-type=git, maintainer=Guillaume Abrioux , io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, release=754, io.buildah.version=1.19.8, architecture=x86_64, GIT_BRANCH=HEAD, name=centos-stream) 2024-05-20T22:43:21.534 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:21 smithi055 bash[62937]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y 2024-05-20T22:43:21.607 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:43:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:43:21] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-20T22:43:21.795 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:21 smithi055 ceph-mon[24757]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:21.795 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:21 smithi055 ceph-mon[29498]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:21.796 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:21 smithi055 podman[62949]: 2024-05-20 22:43:21.541494058 +0000 UTC m=+0.511558375 container remove 70d348a53fb955a3abed20c7ca388d7651966e06565a4e0b84ee14c02c9cb8c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, release=754, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.openshift.expose-services=, vcs-type=git, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc.) 2024-05-20T22:43:21.796 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:21 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-05-20T22:43:22.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:21 smithi138 ceph-mon[25673]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:22.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:21 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-05-20T22:43:22.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:21 smithi055 systemd[1]: Stopped Ceph mgr.y for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:43:22.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:21 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.y.service: Consumed 1min 3.930s CPU time. 2024-05-20T22:43:22.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 systemd[1]: Starting Ceph mgr.y for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:43:22.492 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 podman[63076]: 2024-05-20 22:43:22.144593662 +0000 UTC m=+0.017782845 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:43:22.492 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 podman[63076]: 2024-05-20 22:43:22.243161398 +0000 UTC m=+0.116350626 container create 8e45ecee526afeda5089451cd01774902c67577e6e44eff8cf37b01e60146b25 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514) 2024-05-20T22:43:22.492 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 podman[63076]: 2024-05-20 22:43:22.471400847 +0000 UTC m=+0.344590035 container init 8e45ecee526afeda5089451cd01774902c67577e6e44eff8cf37b01e60146b25 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-05-20T22:43:22.493 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 podman[63076]: 2024-05-20 22:43:22.474527131 +0000 UTC m=+0.347716320 container start 8e45ecee526afeda5089451cd01774902c67577e6e44eff8cf37b01e60146b25 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5) 2024-05-20T22:43:22.838 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 ceph-mgr[63095]: -- 172.21.15.55:0/2951488651 <== mon.1 v2:172.21.15.55:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d552f325a0 con 0x55d552f07000 2024-05-20T22:43:22.838 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 bash[63076]: 8e45ecee526afeda5089451cd01774902c67577e6e44eff8cf37b01e60146b25 2024-05-20T22:43:22.838 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 systemd[1]: Started Ceph mgr.y for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:43:22.838 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:22.642+0000 7fe1221bd200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:43:22.838 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:22.844+0000 7fe1221bd200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:43:23.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:22.955+0000 7fe1221bd200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:43:23.496 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:23.310+0000 7fe1221bd200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:43:23.497 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:23.403+0000 7fe1221bd200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:43:23.857 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:23.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:23 smithi138 ceph-mon[25673]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:23.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:23 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:23 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:23 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:23 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:23 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:43:23.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[24757]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:23.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:43:23.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[29498]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:23.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:23.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:23 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:43:25.172 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:24 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:24.798+0000 7fe1221bd200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:43:25.173 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:25 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:24.999+0000 7fe1221bd200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:43:25.173 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:25 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:25.179+0000 7fe1221bd200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:43:25.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:25 smithi055 ceph-mon[24757]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:25.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:25 smithi055 ceph-mon[29498]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:25.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:25 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:25.496+0000 7fe1221bd200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:43:25.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:25 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:25.595+0000 7fe1221bd200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:43:26.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:25 smithi138 ceph-mon[25673]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:26.131 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:25 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:25.809+0000 7fe1221bd200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:43:26.131 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:26.021+0000 7fe1221bd200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:43:26.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:26 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:26.678+0000 7fe1221bd200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:43:27.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:27.151+0000 7fe1221bd200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:43:27.643 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:27.254+0000 7fe1221bd200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:43:27.643 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:27.488+0000 7fe1221bd200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T22:43:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[29498]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-20T22:43:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:43:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:43:27.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:27.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:27.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[24757]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-20T22:43:27.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:27.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:27.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:43:27.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:43:27.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:27 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:28.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:27 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:28.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:27 smithi138 ceph-mon[25673]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-20T22:43:28.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:27 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:28.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:27 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:28.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:27 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:43:28.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:27 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:43:28.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:27 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:43:28.398 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:28 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:28.042+0000 7fe1221bd200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:43:28.398 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:28 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:28.141+0000 7fe1221bd200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:43:28.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:28 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:28.404+0000 7fe1221bd200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:43:28.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:28 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:28.549+0000 7fe1221bd200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:43:28.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:28 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:28.652+0000 7fe1221bd200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:43:29.086 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:28 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:28.777+0000 7fe1221bd200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:43:29.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:29.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:29 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:29.092+0000 7fe1221bd200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:43:29.964 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:29 smithi055 ceph-mon[24757]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-20T22:43:29.965 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:29 smithi055 ceph-mon[29498]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-20T22:43:29.965 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:29 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:29.738+0000 7fe1221bd200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:43:30.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:29 smithi138 ceph-mon[25673]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-20T22:43:30.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:29 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:29.970+0000 7fe1221bd200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:43:30.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:29 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:43:29] ENGINE Bus STARTING 2024-05-20T22:43:30.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CherryPy Checker: 2024-05-20T22:43:30.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: The Application mounted at '' has an empty config. 2024-05-20T22:43:30.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:43:30.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:43:30] ENGINE Serving on http://:::9283 2024-05-20T22:43:30.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:43:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:43:30] ENGINE Bus STARTED 2024-05-20T22:43:30.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[29498]: Standby manager daemon y restarted 2024-05-20T22:43:30.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[29498]: Standby manager daemon y started 2024-05-20T22:43:30.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T22:43:30.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:43:30.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T22:43:30.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:43:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[24757]: Standby manager daemon y restarted 2024-05-20T22:43:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[24757]: Standby manager daemon y started 2024-05-20T22:43:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T22:43:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:43:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T22:43:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:30 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:43:31.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:30 smithi138 ceph-mon[25673]: Standby manager daemon y restarted 2024-05-20T22:43:31.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:30 smithi138 ceph-mon[25673]: Standby manager daemon y started 2024-05-20T22:43:31.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:30 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T22:43:31.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:30 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:43:31.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:30 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T22:43:31.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:30 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:43:31.607 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:43:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:43:31] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-20T22:43:31.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:31 smithi055 ceph-mon[24757]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-20T22:43:31.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:31 smithi055 ceph-mon[24757]: mgrmap e27: x(active, since 3m), standbys: y 2024-05-20T22:43:31.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:31 smithi055 ceph-mon[29498]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-20T22:43:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:31 smithi055 ceph-mon[29498]: mgrmap e27: x(active, since 3m), standbys: y 2024-05-20T22:43:32.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:31 smithi138 ceph-mon[25673]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-20T22:43:32.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:31 smithi138 ceph-mon[25673]: mgrmap e27: x(active, since 3m), standbys: y 2024-05-20T22:43:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:33 smithi138 ceph-mon[25673]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:33.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:33.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:33 smithi055 ceph-mon[24757]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:33.739 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:33 smithi055 ceph-mon[29498]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:34.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:34 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:34 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:34.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:34 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:35.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:35 smithi138 ceph-mon[25673]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:35.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:35 smithi055 ceph-mon[24757]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:35.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:35 smithi055 ceph-mon[29498]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:37 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:37 smithi138 ceph-mon[25673]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:37.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:37 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:37.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:37 smithi055 ceph-mon[24757]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:37.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:37 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:37.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:37 smithi055 ceph-mon[29498]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:43:39.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:39.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:39 smithi138 ceph-mon[25673]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:39.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:39 smithi055 ceph-mon[24757]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:39 smithi055 ceph-mon[29498]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:41.558 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:43:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:43:41] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-05-20T22:43:41.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:41 smithi138 ceph-mon[25673]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:41.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:41 smithi055 ceph-mon[24757]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:41.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:41 smithi055 ceph-mon[29498]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:43.659 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:43 smithi138 ceph-mon[25673]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:43.659 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:43 smithi055 ceph-mon[24757]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:43.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:43 smithi055 ceph-mon[29498]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:45.823 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:45 smithi138 ceph-mon[25673]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:45.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:45 smithi055 ceph-mon[24757]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:45.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:45 smithi055 ceph-mon[29498]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:47.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:47 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:47.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:47 smithi138 ceph-mon[25673]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:47.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:47 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:47.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:47 smithi055 ceph-mon[29498]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:47.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:47 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:47.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:47 smithi055 ceph-mon[24757]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:49.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:48 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:49.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:49 smithi138 ceph-mon[25673]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:49.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:49 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:49.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:49 smithi055 ceph-mon[29498]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:49.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:49 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:49.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:49 smithi055 ceph-mon[24757]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:49.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:49 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:43:51.506 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:43:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:43:51] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-05-20T22:43:51.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:51 smithi138 ceph-mon[25673]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:51.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:51 smithi055 ceph-mon[29498]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:51.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:51 smithi055 ceph-mon[24757]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:53.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:53 smithi138 ceph-mon[25673]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:53.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:53.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:53 smithi055 ceph-mon[29498]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:53.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:53 smithi055 ceph-mon[24757]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:55.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:55 smithi138 ceph-mon[25673]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:55.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:55 smithi055 ceph-mon[24757]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:55.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:55 smithi055 ceph-mon[29498]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:57.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:57 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:57.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:57 smithi138 ceph-mon[25673]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:57 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:57 smithi055 ceph-mon[29498]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:57 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:43:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:57 smithi055 ceph-mon[24757]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:43:59.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:43:58 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:43:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:43:59.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:43:59 smithi138 ceph-mon[25673]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:43:59 smithi055 ceph-mon[24757]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:43:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:43:59 smithi055 ceph-mon[29498]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:01.507 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:44:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:44:01] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-05-20T22:44:01.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:01 smithi138 ceph-mon[25673]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:01.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:01 smithi055 ceph-mon[24757]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:01.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:01 smithi055 ceph-mon[29498]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:03.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:03 smithi138 ceph-mon[25673]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:44:03.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:03 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:03.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:03 smithi055 ceph-mon[24757]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:44:03.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:03 smithi055 ceph-mon[29498]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:44:04.526 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:04 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:04.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:04 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:04.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:04 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:05 smithi138 ceph-mon[25673]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:44:05.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:05 smithi055 ceph-mon[24757]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:44:05.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:05 smithi055 ceph-mon[29498]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:44:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:07 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:07 smithi138 ceph-mon[25673]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:07.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:07 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:07.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:07 smithi055 ceph-mon[24757]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:07.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:07 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:07.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:07 smithi055 ceph-mon[29498]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:09.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:09.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:09 smithi138 ceph-mon[25673]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:44:09.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:09 smithi055 ceph-mon[24757]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:44:09.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:09 smithi055 ceph-mon[29498]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:44:11.562 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:44:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:44:11] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-05-20T22:44:11.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:11 smithi138 ceph-mon[25673]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:11.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:11 smithi055 ceph-mon[24757]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:11.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:11 smithi055 ceph-mon[29498]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:13.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:13 smithi138 ceph-mon[25673]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:13.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:13.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:13 smithi055 ceph-mon[24757]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:13.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:13 smithi055 ceph-mon[29498]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:15.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:15 smithi138 ceph-mon[25673]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:15.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:15 smithi055 ceph-mon[24757]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:15 smithi055 ceph-mon[29498]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:17.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:17 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:17.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:17 smithi138 ceph-mon[25673]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:17.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:17 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:17.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:17 smithi055 ceph-mon[24757]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:17.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:17 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:17.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:17 smithi055 ceph-mon[29498]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:19.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:19.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:19 smithi138 ceph-mon[25673]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:19.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:19.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:19 smithi055 ceph-mon[29498]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:19.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:19.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:19 smithi055 ceph-mon[24757]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:19.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:21.534 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:44:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:44:21] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-05-20T22:44:21.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:21 smithi138 ceph-mon[25673]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:21.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:21 smithi055 ceph-mon[24757]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:21.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:21 smithi055 ceph-mon[29498]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:23.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:23 smithi138 ceph-mon[25673]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:23.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:23.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:23 smithi055 ceph-mon[24757]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:23.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:23 smithi055 ceph-mon[29498]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:25.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:25 smithi138 ceph-mon[25673]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:25.941 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:25 smithi055 ceph-mon[24757]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:25.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:25 smithi055 ceph-mon[29498]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:27 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:27 smithi138 ceph-mon[25673]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:27.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:27 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:27.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:27 smithi055 ceph-mon[24757]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:27 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:27 smithi055 ceph-mon[29498]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:28.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:28 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:44:28.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:28 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:44:28.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:28 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:44:28.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:28 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:44:28.858 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:28.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:28 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:44:28.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:28 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:44:28.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:28 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:44:28.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:28 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:44:28.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:28 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:44:28.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:28 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:44:28.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:28 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:44:28.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:28 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:44:29.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:29 smithi138 ceph-mon[25673]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:29.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:29 smithi055 ceph-mon[24757]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:29.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:29 smithi055 ceph-mon[29498]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:31.516 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:44:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:44:31] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-05-20T22:44:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:31 smithi138 ceph-mon[25673]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:31.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:31 smithi055 ceph-mon[24757]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:31.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:31 smithi055 ceph-mon[29498]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:33.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:33 smithi055 ceph-mon[24757]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:33.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:33 smithi055 ceph-mon[29498]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:33.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:33 smithi138 ceph-mon[25673]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:33.857 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:34.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:34 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:34 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:34.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:34 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:35.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:35 smithi055 ceph-mon[24757]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:35.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:35 smithi055 ceph-mon[29498]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:35.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:35 smithi138 ceph-mon[25673]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:37 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:37 smithi138 ceph-mon[25673]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:37.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:37 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:37.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:37 smithi055 ceph-mon[29498]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:37.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:37 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:37.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:37 smithi055 ceph-mon[24757]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:39.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:39.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:39 smithi138 ceph-mon[25673]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:39.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:39 smithi055 ceph-mon[24757]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:39 smithi055 ceph-mon[29498]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:41.563 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:44:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:44:41] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-20T22:44:41.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:41 smithi138 ceph-mon[25673]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:41.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:41 smithi055 ceph-mon[24757]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:41.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:41 smithi055 ceph-mon[29498]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:43.659 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:43 smithi138 ceph-mon[25673]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:43.659 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:43 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:43 smithi055 ceph-mon[24757]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:43.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:43 smithi055 ceph-mon[29498]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:45.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:45 smithi138 ceph-mon[25673]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:45.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:45 smithi055 ceph-mon[24757]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:45.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:45 smithi055 ceph-mon[29498]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:47.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:47 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:47.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:47 smithi138 ceph-mon[25673]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:47.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:47 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:47.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:47 smithi055 ceph-mon[29498]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:47.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:47 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:47.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:47 smithi055 ceph-mon[24757]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:49.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:48 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:49.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:49 smithi138 ceph-mon[25673]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:49.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:49 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:49.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:49 smithi055 ceph-mon[29498]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:49.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:49 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:49.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:49 smithi055 ceph-mon[24757]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:49.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:49 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:44:51.543 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:44:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:44:51] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-20T22:44:51.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:51 smithi138 ceph-mon[25673]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:51.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:51 smithi055 ceph-mon[24757]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:51.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:51 smithi055 ceph-mon[29498]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:53.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:53 smithi055 ceph-mon[24757]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:53.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:53 smithi055 ceph-mon[29498]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:53.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:53 smithi138 ceph-mon[25673]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:53.857 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:53 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:55.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:55 smithi138 ceph-mon[25673]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:55.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:55 smithi055 ceph-mon[24757]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:55.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:55 smithi055 ceph-mon[29498]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:57.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:57 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:57.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:57 smithi138 ceph-mon[25673]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:57 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:57 smithi055 ceph-mon[24757]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:57 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:44:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:57 smithi055 ceph-mon[29498]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:44:59.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:44:58 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:44:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:44:59.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:44:59 smithi138 ceph-mon[25673]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:44:59 smithi055 ceph-mon[24757]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:44:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:44:59 smithi055 ceph-mon[29498]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:01.531 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:45:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:45:01] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-20T22:45:01.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:01 smithi138 ceph-mon[25673]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:01.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:01 smithi055 ceph-mon[24757]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:01.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:01 smithi055 ceph-mon[29498]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:03.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:03 smithi055 ceph-mon[29498]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:03.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:03 smithi055 ceph-mon[24757]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:03.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:03 smithi138 ceph-mon[25673]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:03.857 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:45:03 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:45:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:45:04.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:04 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:04 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:04.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:04 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:05 smithi138 ceph-mon[25673]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:05.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:05 smithi055 ceph-mon[24757]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:05.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:05 smithi055 ceph-mon[29498]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:07 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:07 smithi138 ceph-mon[25673]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:07.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:07 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:07.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:07 smithi055 ceph-mon[24757]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:07.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:07 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:07.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:07 smithi055 ceph-mon[29498]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:09.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:45:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:45:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:45:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:09 smithi138 ceph-mon[25673]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:09.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:09 smithi055 ceph-mon[24757]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:09.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:09 smithi055 ceph-mon[29498]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:11.571 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:45:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:45:11] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-20T22:45:11.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:11 smithi138 ceph-mon[25673]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:11.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:11 smithi055 ceph-mon[24757]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:11.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:11 smithi055 ceph-mon[29498]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:13.659 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:13 smithi138 ceph-mon[25673]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:13.660 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:45:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:45:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi055\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", domainname=\"(none)\", instance=\"smithi055\", job=\"node\", machine=\"x86_64\", nodename=\"smithi055\", release=\"5.14.0-447.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:45:13.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:13 smithi055 ceph-mon[24757]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:13.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:13 smithi055 ceph-mon[29498]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:15 smithi138 ceph-mon[25673]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:15.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:15 smithi055 ceph-mon[24757]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:15 smithi055 ceph-mon[29498]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:17.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:17 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:17.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:17 smithi138 ceph-mon[25673]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:17.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:17 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:17.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:17 smithi055 ceph-mon[24757]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:17.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:17 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:17.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:17 smithi055 ceph-mon[29498]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:19.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:45:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:45:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd0bace8-16f7-11ef-bc9a-c7b262605968\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-20T22:45:19.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:19 smithi138 ceph-mon[25673]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:19.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:19.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:19 smithi055 ceph-mon[24757]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:19.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:19.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:19 smithi055 ceph-mon[29498]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:19.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:21.550 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:45:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:45:21] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-05-20T22:45:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:21 smithi138 ceph-mon[25673]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:21.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:21 smithi055 ceph-mon[24757]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:21.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:21 smithi055 ceph-mon[29498]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:23.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:23 smithi055 ceph-mon[24757]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:23.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:23 smithi055 ceph-mon[29498]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:23.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:23 smithi138 ceph-mon[25673]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:25.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:25 smithi138 ceph-mon[25673]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:25.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:25 smithi055 ceph-mon[24757]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:25.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:25 smithi055 ceph-mon[29498]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:27 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:27 smithi138 ceph-mon[25673]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:27.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:27 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:27.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:27 smithi055 ceph-mon[24757]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:27 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:27.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:27 smithi055 ceph-mon[29498]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:28.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:28 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:45:28.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:28 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:45:28.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:28 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:45:29.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:29 smithi138 ceph-mon[25673]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:29.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:29 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:45:29.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:29 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:45:29.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:29 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:45:29.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:29 smithi055 ceph-mon[24757]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:29.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:29 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:45:29.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:29 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:45:29.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:29 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:45:29.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:29 smithi055 ceph-mon[29498]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:29.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:29 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:45:29.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:29 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:45:29.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:29 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:45:31.543 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:45:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:45:31] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-05-20T22:45:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:31 smithi138 ceph-mon[25673]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:31.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:31 smithi055 ceph-mon[24757]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:31.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:31 smithi055 ceph-mon[29498]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:33.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:33 smithi055 ceph-mon[24757]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:33.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:33 smithi055 ceph-mon[29498]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:33.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:33 smithi138 ceph-mon[25673]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:34.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:34 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:34 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:34.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:34 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:35.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:35 smithi138 ceph-mon[25673]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:35.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:35 smithi055 ceph-mon[24757]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:35.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:35 smithi055 ceph-mon[29498]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:37 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:37.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:37 smithi138 ceph-mon[25673]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:37.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:37 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:37.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:37 smithi055 ceph-mon[24757]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:37.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:37 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:37.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:37 smithi055 ceph-mon[29498]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:39.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:39 smithi138 ceph-mon[25673]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:39.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:39 smithi055 ceph-mon[24757]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:39 smithi055 ceph-mon[29498]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:41.559 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:45:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:45:41] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-20T22:45:41.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:41 smithi138 ceph-mon[25673]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:41.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:41 smithi055 ceph-mon[24757]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:41.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:41 smithi055 ceph-mon[29498]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:43.659 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:43 smithi138 ceph-mon[25673]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:43 smithi055 ceph-mon[24757]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:43.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:43 smithi055 ceph-mon[29498]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:45.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:45 smithi138 ceph-mon[25673]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:45.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:45 smithi055 ceph-mon[24757]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:45.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:45 smithi055 ceph-mon[29498]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:47.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:47 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:47.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:47 smithi138 ceph-mon[25673]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:47.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:47 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:47.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:47 smithi055 ceph-mon[24757]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:47.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:47 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:47.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:47 smithi055 ceph-mon[29498]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:49.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:49 smithi138 ceph-mon[25673]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:49.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:49 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:49.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:49 smithi055 ceph-mon[24757]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:49.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:49 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:49.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:49 smithi055 ceph-mon[29498]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:49.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:49 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:45:51.576 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:45:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:45:51] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-05-20T22:45:51.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:51 smithi138 ceph-mon[25673]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:51.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:51 smithi055 ceph-mon[24757]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:51.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:51 smithi055 ceph-mon[29498]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:53.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:53 smithi055 ceph-mon[24757]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:53.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:53 smithi055 ceph-mon[29498]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:53.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:53 smithi138 ceph-mon[25673]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:55.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:55 smithi138 ceph-mon[25673]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:55.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:55 smithi055 ceph-mon[24757]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:55.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:55 smithi055 ceph-mon[29498]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:57.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:57 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:57.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:57 smithi138 ceph-mon[25673]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:57 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:57 smithi055 ceph-mon[24757]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:57 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:45:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:57 smithi055 ceph-mon[29498]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:45:59.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:45:59 smithi138 ceph-mon[25673]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:45:59 smithi055 ceph-mon[24757]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:45:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:45:59 smithi055 ceph-mon[29498]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:01.557 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:46:01] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-05-20T22:46:01.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:01 smithi138 ceph-mon[25673]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:01.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:01 smithi055 ceph-mon[24757]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:01.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:01 smithi055 ceph-mon[29498]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:03.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:03 smithi055 ceph-mon[24757]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:03.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:03 smithi055 ceph-mon[29498]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:03.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:03 smithi138 ceph-mon[25673]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:04 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:04.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:04 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:04 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:05.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:05 smithi138 ceph-mon[25673]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:05.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:05 smithi055 ceph-mon[24757]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:05.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:05 smithi055 ceph-mon[29498]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:07.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:07 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:07.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:07 smithi138 ceph-mon[25673]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:07.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:07 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:07.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:07 smithi055 ceph-mon[24757]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:07.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:07 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:07.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:07 smithi055 ceph-mon[29498]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:09 smithi138 ceph-mon[25673]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:09.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:09 smithi055 ceph-mon[24757]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:09.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:09 smithi055 ceph-mon[29498]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:11.569 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:46:11] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-20T22:46:11.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:11 smithi138 ceph-mon[25673]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:11.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:11 smithi055 ceph-mon[24757]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:11.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:11 smithi055 ceph-mon[29498]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:13.659 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:13 smithi138 ceph-mon[25673]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:13.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:13 smithi055 ceph-mon[24757]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:13.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:13 smithi055 ceph-mon[29498]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:15 smithi138 ceph-mon[25673]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:15.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:15 smithi055 ceph-mon[24757]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:15 smithi055 ceph-mon[29498]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:17.163 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T22:46:17.785 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:17 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:17.785 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:17 smithi055 ceph-mon[24757]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:17.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:17 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:17.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:17 smithi055 ceph-mon[29498]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:17.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:17 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:17.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:17 smithi138 ceph-mon[25673]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:19.148 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:46:19.148 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (5m) 2m ago 13m 23.1M - 0.23.0 ba2b418f427c a0f540c0ca04 2024-05-20T22:46:19.148 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (5m) 3m ago 13m 41.1M - 8.3.5 dad864ee21e9 179b97bd7400 2024-05-20T22:46:19.148 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (5m) 2m ago 12m 48.8M - 3.5 e1d6a67b021e 0f3d86f87e62 2024-05-20T22:46:19.148 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283 running (8m) 3m ago 18m 514M - 19.0.0-2695-g8a373da3 86045c3f4e75 0639e81ef3c8 2024-05-20T22:46:19.148 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (2m) 2m ago 20m 66.1M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (20m) 2m ago 20m 60.4M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (18m) 3m ago 18m 50.2M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (18m) 2m ago 18m 50.3M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (6m) 2m ago 14m 19.9M - 1.3.1 1dbe0e931976 69655ac4483b 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (5m) 3m ago 14m 16.8M - 1.3.1 1dbe0e931976 c17852f7fcd9 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (18m) 2m ago 18m 53.1M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (17m) 2m ago 17m 53.7M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (17m) 2m ago 17m 51.4M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (16m) 2m ago 16m 51.5M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (16m) 3m ago 16m 51.4M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (16m) 3m ago 16m 49.9M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (15m) 3m ago 15m 48.7M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (15m) 3m ago 15m 50.7M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:46:19.149 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (5m) 3m ago 14m 70.5M - 2.33.4 514e6a882f6e f5956ab8bbba 2024-05-20T22:46:19.150 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (13m) 2m ago 13m 71.7M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:46:19.150 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (12m) 3m ago 12m 74.9M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:46:19.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:19 smithi055 ceph-mon[29498]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:19.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:19 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:19.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:19 smithi055 ceph-mon[24757]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:19.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:19 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:19.790 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions' 2024-05-20T22:46:19.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:19 smithi138 ceph-mon[25673]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:19.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:19 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:20.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:20 smithi055 ceph-mon[29498]: from='client.15186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:46:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:20 smithi055 ceph-mon[24757]: from='client.15186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:46:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:20 smithi138 ceph-mon[25673]: from='client.15186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:46:21.581 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:46:21] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-20T22:46:21.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:21 smithi055 ceph-mon[24757]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:21.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:21 smithi055 ceph-mon[29498]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:21 smithi138 ceph-mon[25673]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:21.958 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:46:21.958 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:46:21.958 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:46:21.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:46:21.958 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:46:21.958 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:46:21.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:46:21.959 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:46:21.959 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:46:21.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:46:21.959 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:46:21.959 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:46:21.960 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:46:21.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:46:21.960 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:46:21.960 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:46:21.960 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:46:21.960 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:46:21.960 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:46:22.594 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T22:46:22.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:22 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1892121912' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:46:22.915 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:22 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1892121912' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:46:22.915 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:22 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1892121912' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:46:23.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:23 smithi138 ceph-mon[25673]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:23.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:23 smithi055 ceph-mon[24757]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:23.863 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:23 smithi055 ceph-mon[29498]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:24.693 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T22:46:25.300 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph -s' 2024-05-20T22:46:25.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:25 smithi138 ceph-mon[25673]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:25.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:25 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1874749521' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:46:25.874 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:25 smithi055 ceph-mon[24757]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:25.874 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:25 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1874749521' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:46:25.874 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:25 smithi055 ceph-mon[29498]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:25.874 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:25 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1874749521' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:46:27.352 INFO:teuthology.orchestra.run.smithi055.stdout: cluster: 2024-05-20T22:46:27.352 INFO:teuthology.orchestra.run.smithi055.stdout: id: dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:46:27.352 INFO:teuthology.orchestra.run.smithi055.stdout: health: HEALTH_OK 2024-05-20T22:46:27.352 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: services: 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: mgr: x(active, since 6m), standbys: y 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: data: 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: pools: 6 pools, 161 pgs 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: objects: 209 objects, 585 KiB 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: pgs: 161 active+clean 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:46:27.353 INFO:teuthology.orchestra.run.smithi055.stdout: io: 2024-05-20T22:46:27.354 INFO:teuthology.orchestra.run.smithi055.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-05-20T22:46:27.354 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:46:27.603 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:27 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:27 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:27 smithi138 ceph-mon[25673]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:27.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:27 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3128507529' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:46:27.952 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:27 smithi055 ceph-mon[24757]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:27.953 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:27 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3128507529' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:46:27.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:27 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:27.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:27 smithi055 ceph-mon[29498]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:46:27.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:27 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3128507529' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:46:27.979 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph mgr fail' 2024-05-20T22:46:29.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:29 smithi055 ceph-mon[24757]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:29.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:29 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:29.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:29 smithi055 ceph-mon[29498]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:29.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:29 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:30.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:29 smithi138 ceph-mon[25673]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:46:30.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:29 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:30.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:30 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:30.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:30 smithi138 ceph-mon[25673]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:46:30.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:30 smithi138 ceph-mon[25673]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:46:30.890 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:30 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/999207701' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:46:30.891 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:30 smithi138 ceph-mon[25673]: osdmap e105: 8 total, 8 up, 8 in 2024-05-20T22:46:30.891 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ignoring --setuser ceph since I am not root 2024-05-20T22:46:30.891 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ignoring --setgroup ceph since I am not root 2024-05-20T22:46:30.891 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:30 smithi138 ceph-mgr[45761]: -- 172.21.15.138:0/998509835 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b7fd31c5a0 con 0x55b7fd2f0800 2024-05-20T22:46:30.891 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:30.891 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[29498]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:46:30.891 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[29498]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:46:30.892 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/999207701' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:46:30.892 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[29498]: osdmap e105: 8 total, 8 up, 8 in 2024-05-20T22:46:30.892 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:30.892 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[24757]: from='mgr.34766 172.21.15.138:0/54637902' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:46:30.892 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[24757]: from='mgr.34766 ' entity='mgr.x' 2024-05-20T22:46:30.892 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/999207701' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:46:30.892 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:30 smithi055 ceph-mon[24757]: osdmap e105: 8 total, 8 up, 8 in 2024-05-20T22:46:30.892 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:46:30] ENGINE Bus STOPPING 2024-05-20T22:46:30.892 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:46:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T22:46:30.893 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:46:30] ENGINE Bus STOPPED 2024-05-20T22:46:30.893 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:46:30] ENGINE Bus STARTING 2024-05-20T22:46:31.143 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:46:30] ENGINE Serving on http://:::9283 2024-05-20T22:46:31.143 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:30 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:46:30] ENGINE Bus STARTED 2024-05-20T22:46:31.212 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:30.917+0000 7f7bd8f49200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:46:31.212 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:31.110+0000 7f7bd8f49200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:46:31.212 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:31.218+0000 7f7bd8f49200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:46:31.426 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'sleep 180' 2024-05-20T22:46:31.541 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:31.399Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.138:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.138:8765: connect: connection refused" 2024-05-20T22:46:31.542 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:31.399Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.138:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.138:8765: connect: connection refused" 2024-05-20T22:46:31.542 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:31.400Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.138:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.138:8765: connect: connection refused" 2024-05-20T22:46:31.542 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:31.402Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.138:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.138:8765: connect: connection refused" 2024-05-20T22:46:31.542 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:31.402Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.138:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.138:8765: connect: connection refused" 2024-05-20T22:46:31.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:31.548+0000 7f7bd8f49200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:46:31.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:31.645+0000 7f7bd8f49200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:46:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/999207701' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-20T22:46:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: mgrmap e28: y(active, starting, since 0.665209s) 2024-05-20T22:46:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:46:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:46:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:46:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:46:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: Manager daemon y is now available 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:31.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:31.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:46:31.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:46:31.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:46:31.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:31 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:46:31.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/999207701' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-20T22:46:31.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: mgrmap e28: y(active, starting, since 0.665209s) 2024-05-20T22:46:31.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:46:31.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:46:31.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: Manager daemon y is now available 2024-05-20T22:46:31.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:31.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:31.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:46:31.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:46:31.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:46:31.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/999207701' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: mgrmap e28: y(active, starting, since 0.665209s) 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:46:31.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: Manager daemon y is now available 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:46:31.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:31 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:46:32.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:32 smithi138 ceph-mon[25673]: mgrmap e29: y(active, since 1.6759s) 2024-05-20T22:46:32.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:32 smithi138 ceph-mon[25673]: [20/May/2024:22:46:32] ENGINE Bus STARTING 2024-05-20T22:46:32.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:32 smithi138 ceph-mon[25673]: [20/May/2024:22:46:32] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:46:33.100 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:32 smithi055 ceph-mon[24757]: mgrmap e29: y(active, since 1.6759s) 2024-05-20T22:46:33.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:32 smithi055 ceph-mon[24757]: [20/May/2024:22:46:32] ENGINE Bus STARTING 2024-05-20T22:46:33.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:32 smithi055 ceph-mon[24757]: [20/May/2024:22:46:32] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:46:33.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:32 smithi055 ceph-mon[29498]: mgrmap e29: y(active, since 1.6759s) 2024-05-20T22:46:33.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:32 smithi055 ceph-mon[29498]: [20/May/2024:22:46:32] ENGINE Bus STARTING 2024-05-20T22:46:33.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:32 smithi055 ceph-mon[29498]: [20/May/2024:22:46:32] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:46:33.288 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:32.904+0000 7f7bd8f49200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:46:33.289 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:33.097+0000 7f7bd8f49200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:46:33.289 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:33.295+0000 7f7bd8f49200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:46:33.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:33 smithi138 ceph-mon[25673]: [20/May/2024:22:46:32] ENGINE Serving on http://172.21.15.55:8765 2024-05-20T22:46:33.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:33 smithi138 ceph-mon[25673]: [20/May/2024:22:46:32] ENGINE Bus STARTED 2024-05-20T22:46:33.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:33 smithi138 ceph-mon[25673]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:33.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:33 smithi138 ceph-mon[25673]: mgrmap e30: y(active, since 2s) 2024-05-20T22:46:33.857 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:33.603+0000 7f7bd8f49200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:46:33.857 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:33.699+0000 7f7bd8f49200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:46:33.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:33 smithi055 ceph-mon[24757]: [20/May/2024:22:46:32] ENGINE Serving on http://172.21.15.55:8765 2024-05-20T22:46:33.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:33 smithi055 ceph-mon[24757]: [20/May/2024:22:46:32] ENGINE Bus STARTED 2024-05-20T22:46:33.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:33 smithi055 ceph-mon[24757]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:33.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:33 smithi055 ceph-mon[24757]: mgrmap e30: y(active, since 2s) 2024-05-20T22:46:33.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:33 smithi055 ceph-mon[29498]: [20/May/2024:22:46:32] ENGINE Serving on http://172.21.15.55:8765 2024-05-20T22:46:33.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:33 smithi055 ceph-mon[29498]: [20/May/2024:22:46:32] ENGINE Bus STARTED 2024-05-20T22:46:33.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:33 smithi055 ceph-mon[29498]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:33.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:33 smithi055 ceph-mon[29498]: mgrmap e30: y(active, since 2s) 2024-05-20T22:46:34.229 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:33.908+0000 7f7bd8f49200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:46:34.229 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:34.120+0000 7f7bd8f49200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:46:35.106 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:34.752+0000 7f7bd8f49200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:46:35.527 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:35.204+0000 7f7bd8f49200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:46:35.527 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:35.304+0000 7f7bd8f49200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:46:35.527 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:35.534+0000 7f7bd8f49200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T22:46:35.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:35 smithi138 ceph-mon[25673]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:35.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:35 smithi138 ceph-mon[25673]: mgrmap e31: y(active, since 4s) 2024-05-20T22:46:35.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:35 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:35.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:35 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:35.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:35 smithi055 ceph-mon[24757]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:35.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:35 smithi055 ceph-mon[24757]: mgrmap e31: y(active, since 4s) 2024-05-20T22:46:35.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:35 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:35.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:35 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:35.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:35 smithi055 ceph-mon[29498]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:35.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:35 smithi055 ceph-mon[29498]: mgrmap e31: y(active, since 4s) 2024-05-20T22:46:35.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:35 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:35.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:35 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:36.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:36.086+0000 7f7bd8f49200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:46:36.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:36.186+0000 7f7bd8f49200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:46:36.770 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:36.449+0000 7f7bd8f49200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:46:36.770 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:36.596+0000 7f7bd8f49200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:46:36.771 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:36.695+0000 7f7bd8f49200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:46:37.106 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:36.815+0000 7f7bd8f49200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:46:37.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:36 smithi138 ceph-mon[25673]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:37.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:36 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:37.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:36 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:37.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:36 smithi055 ceph-mon[24757]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:37.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:36 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:37.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:36 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:37.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:36 smithi055 ceph-mon[29498]: from='client.34886 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:46:37.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:36 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:37.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:36 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:37.609 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:37.121+0000 7f7bd8f49200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:46:38.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:37 smithi138 ceph-mon[25673]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:38.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:37 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:38.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:37 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:38.057 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:37.733+0000 7f7bd8f49200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:46:38.057 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:37.956+0000 7f7bd8f49200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:46:38.057 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:46:37] ENGINE Bus STARTING 2024-05-20T22:46:38.058 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: CherryPy Checker: 2024-05-20T22:46:38.058 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: The Application mounted at '' has an empty config. 2024-05-20T22:46:38.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:37 smithi055 ceph-mon[24757]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:38.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:37 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:38.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:37 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:38.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:37 smithi055 ceph-mon[29498]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:38.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:37 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:38.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:37 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:38.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:46:38.357 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:46:38] ENGINE Serving on http://:::9283 2024-05-20T22:46:38.357 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:46:38] ENGINE Bus STARTED 2024-05-20T22:46:39.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:46:39.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: Standby manager daemon x started 2024-05-20T22:46:39.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:46:39.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[24757]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: Standby manager daemon x started 2024-05-20T22:46:39.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:46:39.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:38 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:46:39.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: Standby manager daemon x started 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:46:39.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:38 smithi138 ceph-mon[25673]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:46:40.222 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:46:40.222 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: mgrmap e32: y(active, since 8s), standbys: x 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:40.223 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: mgrmap e32: y(active, since 8s), standbys: x 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.225 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.225 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:46:40.225 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:40.225 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:40.225 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:40.225 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:39 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:46:40.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:46:40.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:46:40.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:46:40.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:46:40.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:46:40.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: mgrmap e32: y(active, since 8s), standbys: x 2024-05-20T22:46:40.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:46:40.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:40.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:46:40.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:40.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:40.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:40.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:39 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:46:41.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:46:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:46:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:46:42.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:41 smithi055 ceph-mon[24757]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:46:42.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:41 smithi055 ceph-mon[29498]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:46:42.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:41 smithi138 ceph-mon[25673]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:46:42.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:46:42.350+0000 7fe0a7d2d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:46:42.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:46:42.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:46:42.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:46:42.735 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:46:42.736 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:46:42.736 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:42.736 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:42.736 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:46:42.736 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:46:42.736 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return future.result(timeout) 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.__get_result() 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: raise self._exception 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: use_current_daemon_image=reconfig, 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:46:42.737 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:46:42.738 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:46:42.739 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:46:42.739 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:46:42.739 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:46:42.739 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:46:42.739 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:46:42.739 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:46:42.739 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:42.739 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.457 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:43 smithi138 ceph-mon[25673]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:46:43.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:46:43.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:46:43.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:46:43.457 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:46:43.458 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: return future.result(timeout) 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:46:43.459 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.__get_result() 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: raise self._exception 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.460 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.461 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.462 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:43 smithi138 ceph-mon[25673]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:46:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:43 smithi055 ceph-mon[24757]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:46:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:46:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:46:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:46:43.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:46:43.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: return future.result(timeout) 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.__get_result() 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:46:43.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: raise self._exception 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:43 smithi055 ceph-mon[29498]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:46:43.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:46:43.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: return future.result(timeout) 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.__get_result() 2024-05-20T22:46:43.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: raise self._exception 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:46:43.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.742 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.743 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:43 smithi055 ceph-mon[29498]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:46:43.743 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:46:43.743 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:46:43.743 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:46:43.743 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:46:43.743 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:46:43.743 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:43 smithi055 ceph-mon[24757]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:46:44.537 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:44 smithi138 ceph-mon[25673]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:46:44.537 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:44 smithi138 ceph-mon[25673]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:46:44.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:44 smithi055 ceph-mon[29498]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:46:44.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:44 smithi055 ceph-mon[29498]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:46:44.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:44 smithi055 ceph-mon[24757]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:46:44.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:44 smithi055 ceph-mon[24757]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:46:46.106 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:45 smithi138 systemd[1]: Stopping Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:46:46.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:45 smithi138 bash[54232]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:46:46.418 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.140Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.140Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.140Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.140Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.140Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.140Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.140Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.140Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.140Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.141Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-20T22:46:46.419 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.141Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-20T22:46:46.420 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[48779]: ts=2024-05-20T22:46:46.141Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-20T22:46:46.420 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 podman[54244]: 2024-05-20 22:46:46.150888829 +0000 UTC m=+0.143193411 container died f5956ab8bbba15dc4e7c7ad79bd83eb7d86604476b8114b08f35d2aab612f55a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:46:46.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:46 smithi055 ceph-mon[24757]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:46:46.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:46 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:46.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:46 smithi055 ceph-mon[29498]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:46:46.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:46 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:46.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:46 smithi138 ceph-mon[25673]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:46:46.783 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:46 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:46:46.783 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 podman[54244]: 2024-05-20 22:46:46.523116673 +0000 UTC m=+0.515421258 container cleanup f5956ab8bbba15dc4e7c7ad79bd83eb7d86604476b8114b08f35d2aab612f55a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:46:46.784 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 bash[54244]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a 2024-05-20T22:46:46.784 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 bash[54270]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:46:47.106 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 podman[54257]: 2024-05-20 22:46:46.790910566 +0000 UTC m=+0.637712397 container remove f5956ab8bbba15dc4e7c7ad79bd83eb7d86604476b8114b08f35d2aab612f55a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:46:47.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-20T22:46:47.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 systemd[1]: Stopped Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:46:47.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Consumed 2.026s CPU time. 2024-05-20T22:46:47.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:46 smithi138 systemd[1]: Starting Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:46:47.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 podman[54336]: 2024-05-20 22:46:46.927839388 +0000 UTC m=+0.017688943 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-20T22:46:47.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 podman[54336]: 2024-05-20 22:46:47.033612988 +0000 UTC m=+0.123462552 container create 9aaf823c00efb8e4b087aae35d597010a7d0d991d0811574e053870af7f3b71a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:46:47.475 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 podman[54336]: 2024-05-20 22:46:47.244317458 +0000 UTC m=+0.334167009 container init 9aaf823c00efb8e4b087aae35d597010a7d0d991d0811574e053870af7f3b71a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:46:47.475 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 podman[54336]: 2024-05-20 22:46:47.24698708 +0000 UTC m=+0.336836630 container start 9aaf823c00efb8e4b087aae35d597010a7d0d991d0811574e053870af7f3b71a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:46:47.475 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.270Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-20T22:46:47.475 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.270Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-20T22:46:47.475 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.270Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.270Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi138 (none))" 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.270Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.270Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.272Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.272Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.274Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.275Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.275Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.616µs 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.275Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 bash[54336]: 9aaf823c00efb8e4b087aae35d597010a7d0d991d0811574e053870af7f3b71a 2024-05-20T22:46:47.476 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 systemd[1]: Started Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:46:47.856 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.482Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-05-20T22:46:47.856 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.709Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-05-20T22:46:48.216 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.929Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-05-20T22:46:48.216 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.930Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-05-20T22:46:48.216 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.930Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=49.864µs wal_replay_duration=654.679216ms total_replay_duration=654.746327ms 2024-05-20T22:46:48.216 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.937Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-20T22:46:48.217 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.937Z caller=main.go:947 level=info msg="TSDB started" 2024-05-20T22:46:48.217 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.937Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-20T22:46:48.217 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.961Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.372054ms db_storage=999ns remote_storage=1.697µs web_handler=422ns query_engine=1.199µs scrape=525.848µs scrape_sd=165.432µs notify=18.015µs notify_sd=18.864µs rules=23.089065ms 2024-05-20T22:46:48.217 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:46:47 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:46:47.961Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-20T22:46:48.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:48 smithi138 ceph-mon[25673]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:46:48.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:48 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:48.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:48 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:48.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:48 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:46:48.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:48 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:46:48.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:48 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[24757]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:46:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:46:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:46:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[29498]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:46:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:46:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:46:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:48 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:50.700 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:50 smithi138 ceph-mon[25673]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:46:50.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:50 smithi055 ceph-mon[24757]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:46:50.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:50 smithi055 ceph-mon[29498]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:46:52.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[24757]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-20T22:46:52.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:52.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:52.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:52.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:46:52.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:52.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:52.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:52.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:52.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[29498]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-20T22:46:52.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:52.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:52.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:52.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:46:52.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:52.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:52.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:52.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:52 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:52.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:52 smithi138 ceph-mon[25673]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-20T22:46:52.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:52 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:52.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:52 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:52.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:52 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:52.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:52 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:46:52.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:52 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:52.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:52 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:52.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:52 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:46:52.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:52 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:46:53.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:53 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:46:53.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:53 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:46:53.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:53 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:46:53.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:53 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:46:53.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:53 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:46:53.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:53 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:46:54.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:54 smithi055 ceph-mon[24757]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-20T22:46:54.667 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:54 smithi055 ceph-mon[29498]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-20T22:46:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:54 smithi138 ceph-mon[25673]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-20T22:46:56.707 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:56 smithi055 ceph-mon[24757]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:56.707 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:56 smithi055 ceph-mon[29498]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:56.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:56 smithi138 ceph-mon[25673]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:57.720 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:46:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:46:57] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-05-20T22:46:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[24757]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:46:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:46:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:46:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:46:57.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3527589260' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[29498]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:57 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3527589260' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:46:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:57 smithi138 ceph-mon[25673]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:46:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:57 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:57 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:46:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:57 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:46:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:57 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:46:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:57 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:46:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:57 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:46:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:57 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:46:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:57 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3527589260' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:47:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:46:59 smithi138 ceph-mon[25673]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:47:00.154 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:46:59 smithi055 ceph-mon[24757]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:47:00.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:46:59 smithi055 ceph-mon[29498]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:47:01.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:00 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:01.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:00 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:01.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:00 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:02.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[24757]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:47:02.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:47:02.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:47:02.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:47:02.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:47:02.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:47:02.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[29498]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:47:02.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:47:02.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:47:02.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:47:02.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:47:02.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:02 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:47:02.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:02 smithi138 ceph-mon[25673]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:47:02.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:02 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:47:02.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:02 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:47:02.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:02 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:47:02.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:02 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:47:02.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:02 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:47:04.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:04 smithi055 ceph-mon[24757]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:47:04.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:04 smithi055 ceph-mon[29498]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:47:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:04 smithi138 ceph-mon[25673]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:47:06.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:06 smithi055 ceph-mon[24757]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:47:06.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:06 smithi055 ceph-mon[29498]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:47:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:06 smithi138 ceph-mon[25673]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:47:07.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:47:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:47:07] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-05-20T22:47:08.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:08 smithi055 ceph-mon[24757]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:47:08.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:08 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:08.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:08 smithi055 ceph-mon[29498]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:47:08.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:08 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:08 smithi138 ceph-mon[25673]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:47:08.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:08 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:10.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:10 smithi055 ceph-mon[24757]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:47:10.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:10 smithi055 ceph-mon[29498]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:47:10.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:10 smithi138 ceph-mon[25673]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:47:12.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:12 smithi055 ceph-mon[24757]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:12.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:12 smithi055 ceph-mon[29498]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:12.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:12 smithi138 ceph-mon[25673]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:14.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:14 smithi055 ceph-mon[24757]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:14.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:14 smithi055 ceph-mon[29498]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:14.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:14 smithi138 ceph-mon[25673]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:16.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:16 smithi055 ceph-mon[24757]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:16.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:16 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:16.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:16 smithi055 ceph-mon[29498]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:16.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:16 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:16.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:16 smithi138 ceph-mon[25673]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:16.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:16 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:17.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:47:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:47:17] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-05-20T22:47:18.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:18 smithi055 ceph-mon[24757]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:18.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:18 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:18.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:18 smithi055 ceph-mon[29498]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:18.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:18 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:18.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:18 smithi138 ceph-mon[25673]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:18.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:18 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:20 smithi055 ceph-mon[24757]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:20.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:20 smithi055 ceph-mon[29498]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:20 smithi138 ceph-mon[25673]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:22.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:22 smithi055 ceph-mon[24757]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:22.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:22 smithi055 ceph-mon[29498]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:22.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:22 smithi138 ceph-mon[25673]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:24.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:24 smithi055 ceph-mon[24757]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:24.799 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:24 smithi055 ceph-mon[29498]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:24.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:24 smithi138 ceph-mon[25673]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:26.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:26 smithi138 ceph-mon[25673]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:26.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:26 smithi055 ceph-mon[24757]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:26.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:26 smithi055 ceph-mon[29498]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:27.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:47:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:47:27] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-05-20T22:47:28.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:28 smithi138 ceph-mon[25673]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:28.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:28 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:28.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:28 smithi055 ceph-mon[24757]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:28.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:28 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:28.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:28 smithi055 ceph-mon[29498]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:28.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:28 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:30.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:30 smithi138 ceph-mon[25673]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:30 smithi055 ceph-mon[24757]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:30.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:30 smithi055 ceph-mon[29498]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:31.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:31.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:31.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:32.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:32 smithi138 ceph-mon[25673]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:32.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:32 smithi055 ceph-mon[24757]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:32.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:32 smithi055 ceph-mon[29498]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:33.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:33 smithi138 ceph-mon[25673]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:33.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:33 smithi055 ceph-mon[24757]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:33.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:33 smithi055 ceph-mon[29498]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:36.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:35 smithi138 ceph-mon[25673]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:36.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:35 smithi055 ceph-mon[24757]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:36.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:35 smithi055 ceph-mon[29498]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:37.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:47:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:47:37] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-05-20T22:47:38.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:38 smithi055 ceph-mon[24757]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:38.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:38 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:38.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:38 smithi055 ceph-mon[29498]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:38.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:38 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:38.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:38 smithi138 ceph-mon[25673]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:38.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:38 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:40.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:40 smithi055 ceph-mon[29498]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:40.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:40 smithi055 ceph-mon[24757]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:40 smithi138 ceph-mon[25673]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:42.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:42 smithi055 ceph-mon[24757]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:42.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:42 smithi055 ceph-mon[29498]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:42.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:42 smithi138 ceph-mon[25673]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:44.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:44 smithi055 ceph-mon[24757]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:44.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:44 smithi055 ceph-mon[29498]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:44.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:44 smithi138 ceph-mon[25673]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:46.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:46 smithi055 ceph-mon[24757]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:46.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:46 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:46.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:46 smithi055 ceph-mon[29498]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:46.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:46 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:46.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:46 smithi138 ceph-mon[25673]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:46.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:46 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:47:47.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:47:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:47:47] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-20T22:47:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:48 smithi055 ceph-mon[24757]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:48 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:48.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:48 smithi055 ceph-mon[29498]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:48.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:48 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:48.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:48 smithi138 ceph-mon[25673]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:48.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:48 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:50.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:50 smithi055 ceph-mon[24757]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:50.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:50 smithi055 ceph-mon[29498]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:50.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:50 smithi138 ceph-mon[25673]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:52.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:52 smithi055 ceph-mon[24757]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:52.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:52 smithi055 ceph-mon[29498]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:52.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:52 smithi138 ceph-mon[25673]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:54.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:54 smithi055 ceph-mon[24757]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:54.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:54 smithi055 ceph-mon[29498]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:54 smithi138 ceph-mon[25673]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:47:56.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:56 smithi055 ceph-mon[24757]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:56.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:56 smithi055 ceph-mon[29498]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:56.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:56 smithi138 ceph-mon[25673]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:57.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:47:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:47:57] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-20T22:47:58.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:58 smithi055 ceph-mon[24757]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:58.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:47:58 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:58.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:58 smithi055 ceph-mon[29498]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:58.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:47:58 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:47:58.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:58 smithi138 ceph-mon[25673]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:47:58.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:47:58 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:00.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:00 smithi138 ceph-mon[25673]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:00.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:00 smithi055 ceph-mon[24757]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:00.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:00 smithi055 ceph-mon[29498]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:01.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:01 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:01.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:01 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:01.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:01 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:02.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:02 smithi138 ceph-mon[25673]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:02.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:02 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:48:02.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:02 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:02 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:02 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:02 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[24757]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:02.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:48:02.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[29498]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:02.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:48:02.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:02.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:02 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:03.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:03 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:48:03.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:03 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:48:03.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:03 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:03.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:03 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:48:03.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:03 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:48:03.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:03 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:03.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:03 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:48:03.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:03 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:48:03.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:03 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:48:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:04 smithi138 ceph-mon[25673]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:04.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:04 smithi055 ceph-mon[24757]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:04 smithi055 ceph-mon[29498]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:06 smithi138 ceph-mon[25673]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:06 smithi055 ceph-mon[24757]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:06 smithi055 ceph-mon[29498]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:07.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:48:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:48:07] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-05-20T22:48:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:08 smithi138 ceph-mon[25673]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:08 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:08 smithi055 ceph-mon[24757]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:08 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:08.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:08 smithi055 ceph-mon[29498]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:08.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:08 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:10.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:10 smithi138 ceph-mon[25673]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:10.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:10 smithi055 ceph-mon[24757]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:10.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:10 smithi055 ceph-mon[29498]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:11.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:11 smithi138 ceph-mon[25673]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:11.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:11 smithi055 ceph-mon[24757]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:11.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:11 smithi055 ceph-mon[29498]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:14.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:14 smithi055 ceph-mon[24757]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:14.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:14 smithi055 ceph-mon[29498]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:14.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:14 smithi138 ceph-mon[25673]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:16.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:16 smithi055 ceph-mon[24757]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:16.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:16 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:16.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:16 smithi055 ceph-mon[29498]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:16.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:16 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:16 smithi138 ceph-mon[25673]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:16.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:16 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:17.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:48:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:48:17] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-05-20T22:48:18.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:18 smithi055 ceph-mon[24757]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:18.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:18 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:18.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:18 smithi055 ceph-mon[29498]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:18.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:18 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:18.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:18 smithi138 ceph-mon[25673]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:18.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:18 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:20 smithi055 ceph-mon[24757]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:20.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:20 smithi055 ceph-mon[29498]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:20 smithi138 ceph-mon[25673]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:22.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:22 smithi055 ceph-mon[24757]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:22.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:22 smithi055 ceph-mon[29498]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:22.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:22 smithi138 ceph-mon[25673]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:24.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:24 smithi055 ceph-mon[24757]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:24 smithi055 ceph-mon[29498]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:24.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:24 smithi138 ceph-mon[25673]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:26.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:26 smithi055 ceph-mon[24757]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:26.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:26 smithi055 ceph-mon[29498]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:26.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:26 smithi138 ceph-mon[25673]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:27.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:48:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:48:27] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-05-20T22:48:28.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:28 smithi055 ceph-mon[29498]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:28.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:28 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:28.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:28 smithi055 ceph-mon[24757]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:28.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:28 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:28.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:28 smithi138 ceph-mon[25673]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:28.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:28 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:30.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:30 smithi055 ceph-mon[29498]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:30.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:30 smithi055 ceph-mon[24757]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:30.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:30 smithi138 ceph-mon[25673]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:31.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:31.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:31.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:32.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:32 smithi138 ceph-mon[25673]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:32.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:32 smithi055 ceph-mon[29498]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:32.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:32 smithi055 ceph-mon[24757]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:34.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:34 smithi138 ceph-mon[25673]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:34.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:34 smithi055 ceph-mon[24757]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:34.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:34 smithi055 ceph-mon[29498]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:36 smithi138 ceph-mon[25673]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:36.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:36 smithi055 ceph-mon[24757]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:36.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:36 smithi055 ceph-mon[29498]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:37.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:48:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:48:37] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-20T22:48:38.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:38 smithi138 ceph-mon[25673]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:38.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:38 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:38.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:38 smithi055 ceph-mon[29498]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:38.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:38 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:38.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:38 smithi055 ceph-mon[24757]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:38.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:38 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:40.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:40 smithi138 ceph-mon[25673]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:40.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:40 smithi055 ceph-mon[24757]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:40.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:40 smithi055 ceph-mon[29498]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:42.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:42 smithi138 ceph-mon[25673]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:42.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:42 smithi055 ceph-mon[24757]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:42.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:42 smithi055 ceph-mon[29498]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:44.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:44 smithi138 ceph-mon[25673]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:44.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:44 smithi055 ceph-mon[24757]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:44.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:44 smithi055 ceph-mon[29498]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:45.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:45 smithi138 ceph-mon[25673]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:45.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:45 smithi055 ceph-mon[24757]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:45.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:45 smithi055 ceph-mon[29498]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:46.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:46 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:46.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:46 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:46.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:46 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:48:47.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:48:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:48:47] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-05-20T22:48:48.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:48 smithi055 ceph-mon[24757]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:48 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:48.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:48 smithi055 ceph-mon[29498]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:48.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:48 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:48.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:48 smithi138 ceph-mon[25673]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:48.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:48 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:50.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:50 smithi055 ceph-mon[24757]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:50.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:50 smithi055 ceph-mon[29498]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:50.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:50 smithi138 ceph-mon[25673]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:52.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:52 smithi055 ceph-mon[24757]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:52.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:52 smithi055 ceph-mon[29498]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:52.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:52 smithi138 ceph-mon[25673]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:54.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:54 smithi055 ceph-mon[24757]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:54.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:54 smithi055 ceph-mon[29498]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:54 smithi138 ceph-mon[25673]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:56.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:56 smithi055 ceph-mon[24757]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:56.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:56 smithi055 ceph-mon[29498]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:56.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:56 smithi138 ceph-mon[25673]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:48:57.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:48:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:48:57] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-05-20T22:48:58.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:58 smithi055 ceph-mon[24757]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:58.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:48:58 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:58.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:58 smithi055 ceph-mon[29498]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:58.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:48:58 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:48:58.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:58 smithi138 ceph-mon[25673]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:48:58.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:48:58 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:00.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:00 smithi055 ceph-mon[24757]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:00.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:00 smithi055 ceph-mon[29498]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:00.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:00 smithi138 ceph-mon[25673]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:01.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:01 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:01.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:01 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:01.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:01 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:02.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:02 smithi138 ceph-mon[25673]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:02.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:02 smithi055 ceph-mon[29498]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:02.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:02 smithi055 ceph-mon[24757]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:03.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:03 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:49:03.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:03 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:49:03.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:03 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:49:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:04 smithi138 ceph-mon[25673]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:04 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:49:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:04 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:49:04.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:04 smithi138 ceph-mon[25673]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:49:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:04 smithi055 ceph-mon[24757]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:04.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:04 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:49:04.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:04 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:49:04.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:04 smithi055 ceph-mon[24757]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:49:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:04 smithi055 ceph-mon[29498]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:04 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:49:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:04 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:49:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:04 smithi055 ceph-mon[29498]: from='mgr.15180 ' entity='mgr.y' 2024-05-20T22:49:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:06 smithi138 ceph-mon[25673]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:06 smithi055 ceph-mon[24757]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:06 smithi055 ceph-mon[29498]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:07.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:49:07] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-20T22:49:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:08 smithi138 ceph-mon[25673]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:08 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:08.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:08 smithi055 ceph-mon[24757]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:08 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:08.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:08 smithi055 ceph-mon[29498]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:08.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:08 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:10.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:10 smithi138 ceph-mon[25673]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:10.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:10 smithi055 ceph-mon[24757]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:10.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:10 smithi055 ceph-mon[29498]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:12.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:12 smithi138 ceph-mon[25673]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:12.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:12 smithi055 ceph-mon[24757]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:12.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:12 smithi055 ceph-mon[29498]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:14.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:14 smithi138 ceph-mon[25673]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:14.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:14 smithi055 ceph-mon[24757]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:14.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:14 smithi055 ceph-mon[29498]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:16 smithi138 ceph-mon[25673]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:16 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:16.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:16 smithi055 ceph-mon[29498]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:16 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:16.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:16 smithi055 ceph-mon[24757]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:16.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:16 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:17.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:49:17] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-20T22:49:18.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:18 smithi055 ceph-mon[24757]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:18.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:18 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:18.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:18 smithi055 ceph-mon[29498]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:18.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:18 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:18.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:18 smithi138 ceph-mon[25673]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:18.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:18 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:20 smithi055 ceph-mon[24757]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:20.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:20 smithi055 ceph-mon[29498]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:20 smithi138 ceph-mon[25673]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:22.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:22 smithi055 ceph-mon[24757]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:22.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:22 smithi055 ceph-mon[29498]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:22.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:22 smithi138 ceph-mon[25673]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:24.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:24 smithi055 ceph-mon[24757]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:24 smithi055 ceph-mon[29498]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:24.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:24 smithi138 ceph-mon[25673]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:26.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:26 smithi055 ceph-mon[24757]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:26.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:26 smithi055 ceph-mon[29498]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:26.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:26 smithi138 ceph-mon[25673]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:27.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:49:27] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-20T22:49:28.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:28 smithi055 ceph-mon[24757]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:28.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:28 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:28.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:28 smithi055 ceph-mon[29498]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:28.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:28 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:28.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:28 smithi138 ceph-mon[25673]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:28.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:28 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:30.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:30 smithi055 ceph-mon[24757]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:30.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:30 smithi055 ceph-mon[29498]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:30.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:30 smithi138 ceph-mon[25673]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:31.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:31 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:31.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:31 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:31.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:31 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:32.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:32 smithi055 ceph-mon[24757]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:32.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:32 smithi055 ceph-mon[29498]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:32.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:32 smithi138 ceph-mon[25673]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:34.510 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T22:49:34.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:34 smithi055 ceph-mon[24757]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:34.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:34 smithi055 ceph-mon[29498]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:34.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:34 smithi138 ceph-mon[25673]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:36.651 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:49:36.651 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (9m) 2m ago 16m 22.8M - 0.23.0 ba2b418f427c a0f540c0ca04 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (8m) 2m ago 16m 49.1M - 8.3.5 dad864ee21e9 179b97bd7400 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (2m) 2m ago 16m 45.9M - 3.5 e1d6a67b021e 0a83a23cc0f8 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283 running (11m) 2m ago 21m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 0639e81ef3c8 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (6m) 2m ago 23m 498M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (23m) 2m ago 23m 65.7M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (21m) 2m ago 21m 51.0M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (22m) 2m ago 22m 52.0M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (9m) 2m ago 17m 17.8M - 1.3.1 1dbe0e931976 69655ac4483b 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (9m) 2m ago 17m 18.4M - 1.3.1 1dbe0e931976 c17852f7fcd9 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (21m) 2m ago 21m 54.4M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (20m) 2m ago 20m 54.0M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (20m) 2m ago 20m 52.5M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:49:36.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (20m) 2m ago 20m 51.8M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:49:36.653 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (19m) 2m ago 19m 53.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:49:36.653 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (19m) 2m ago 19m 51.2M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:49:36.653 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (18m) 2m ago 18m 50.3M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:49:36.653 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (18m) 2m ago 18m 52.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:49:36.653 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (2m) 2m ago 17m 43.9M - 2.33.4 514e6a882f6e 9aaf823c00ef 2024-05-20T22:49:36.653 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (16m) 2m ago 16m 73.2M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:49:36.653 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (16m) 2m ago 16m 76.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:49:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:36 smithi138 ceph-mon[25673]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:36.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:36 smithi055 ceph-mon[24757]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:36.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:36 smithi055 ceph-mon[29498]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:37.207 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions' 2024-05-20T22:49:37.460 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:49:37] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-05-20T22:49:38.758 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:38 smithi055 ceph-mon[24757]: from='client.25189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:49:38.758 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:38 smithi055 ceph-mon[24757]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:38.758 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:38 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:38.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:38 smithi055 ceph-mon[29498]: from='client.25189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:49:38.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:38 smithi055 ceph-mon[29498]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:38.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:38 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:38.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:38 smithi138 ceph-mon[25673]: from='client.25189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:49:38.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:38 smithi138 ceph-mon[25673]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:38.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:38 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:49:39.289 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:49:39.290 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:49:39.290 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:49:39.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:49:39.290 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:49:39.290 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:49:39.290 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:49:39.290 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:49:39.290 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:49:39.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:39 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2345742365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:49:39.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:39 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2345742365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:49:39.882 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:39 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2345742365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:49:39.908 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph -s' 2024-05-20T22:49:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:40 smithi138 ceph-mon[25673]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:40.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:40 smithi055 ceph-mon[24757]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:40.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:40 smithi055 ceph-mon[29498]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: cluster: 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: id: dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: health: HEALTH_OK 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: services: 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: mgr: y(active, since 3m), standbys: x 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-20T22:49:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:49:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: data: 2024-05-20T22:49:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: pools: 6 pools, 161 pgs 2024-05-20T22:49:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: objects: 209 objects, 585 KiB 2024-05-20T22:49:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: pgs: 161 active+clean 2024-05-20T22:49:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:49:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: io: 2024-05-20T22:49:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-20T22:49:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-05-20T22:49:42.482 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T22:49:42.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:42 smithi055 ceph-mon[29498]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:42.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:42 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2372134829' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:49:42.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:42 smithi055 ceph-mon[24757]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:42.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:42 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2372134829' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:49:42.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:42 smithi138 ceph-mon[25673]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:42.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:42 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2372134829' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:49:43.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:43 smithi055 ceph-mon[24757]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:43.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:43 smithi055 ceph-mon[29498]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:44.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:43 smithi138 ceph-mon[25673]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:44.624 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T22:49:44.946 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:44 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1173850528' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:49:44.946 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:44 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1173850528' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:49:45.007 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:44 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1173850528' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:49:45.241 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-05-20T22:49:45.875 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:45 smithi055 ceph-mon[24757]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:45.875 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:45 smithi055 ceph-mon[29498]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:46.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:45 smithi138 ceph-mon[25673]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:46.907 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:46 smithi055 ceph-mon[24757]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:46.908 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:46 smithi055 ceph-mon[29498]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:47.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:46 smithi138 ceph-mon[25673]: from='mgr.15180 172.21.15.55:0/3983864576' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:47.268 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:49:47.569 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:49:47] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-05-20T22:49:47.870 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:47 smithi055 ceph-mon[24757]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:47.870 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:47 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:47.870 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:47 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2107797952' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:49:47.870 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:47 smithi055 ceph-mon[29498]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:47.870 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:47 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:47.870 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:47 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2107797952' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:49:47.898 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph mgr fail' 2024-05-20T22:49:48.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:47 smithi138 ceph-mon[25673]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:49:48.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:47 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:48.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:47 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2107797952' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:49:50.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:49 smithi055 ceph-mon[29498]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:50.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:49 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/4275516751' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:49:50.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:49 smithi055 ceph-mon[29498]: osdmap e106: 8 total, 8 up, 8 in 2024-05-20T22:49:50.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:49 smithi055 ceph-mon[24757]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:50.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:49 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4275516751' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:49:50.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:49 smithi055 ceph-mon[24757]: osdmap e106: 8 total, 8 up, 8 in 2024-05-20T22:49:50.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:49 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:49.866+0000 7fe1157af700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-20T22:49:50.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:49 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ignoring --setuser ceph since I am not root 2024-05-20T22:49:50.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:49 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ignoring --setgroup ceph since I am not root 2024-05-20T22:49:50.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:49 smithi055 ceph-mgr[63095]: -- 172.21.15.55:0/2999265742 <== mon.1 v2:172.21.15.55:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ad1fb885a0 con 0x55ad1fb5cc00 2024-05-20T22:49:50.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:50.055+0000 7f362f471200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:49:50.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:49 smithi138 ceph-mon[25673]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:49:50.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:49 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/4275516751' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-20T22:49:50.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:49 smithi138 ceph-mon[25673]: osdmap e106: 8 total, 8 up, 8 in 2024-05-20T22:49:50.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:49:49 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:49:49] ENGINE Bus STOPPING 2024-05-20T22:49:50.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:49:49 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:49:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T22:49:50.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:49:49 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:49:49] ENGINE Bus STOPPED 2024-05-20T22:49:50.358 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:49:49 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:49:49] ENGINE Bus STARTING 2024-05-20T22:49:50.359 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:49:50 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:49:50] ENGINE Serving on http://:::9283 2024-05-20T22:49:50.359 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:49:50 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:49:50] ENGINE Bus STARTED 2024-05-20T22:49:50.546 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:50.251+0000 7f362f471200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:49:50.546 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:50.363+0000 7f362f471200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:49:50.572 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'sleep 180' 2024-05-20T22:49:50.804 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:50.703+0000 7f362f471200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:49:50.804 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:50.811+0000 7f362f471200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:49:51.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/4275516751' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-20T22:49:51.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: mgrmap e33: x(active, starting, since 0.0406853s) 2024-05-20T22:49:51.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:49:51.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:49:51.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:49:51.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: Manager daemon x is now available 2024-05-20T22:49:51.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:49:51.073 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:51.073 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:49:51.073 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:49:51.073 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:49:51.073 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:49:51.073 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: [20/May/2024:22:49:50] ENGINE Bus STARTING 2024-05-20T22:49:51.073 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: [20/May/2024:22:49:50] ENGINE Serving on https://172.21.15.138:7150 2024-05-20T22:49:51.073 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: [20/May/2024:22:49:50] ENGINE Serving on http://172.21.15.138:8765 2024-05-20T22:49:51.073 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[29498]: [20/May/2024:22:49:50] ENGINE Bus STARTED 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/4275516751' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: mgrmap e33: x(active, starting, since 0.0406853s) 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:49:51.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: Manager daemon x is now available 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:49:51.075 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:49:51.076 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:49:51.076 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:49:51.076 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: [20/May/2024:22:49:50] ENGINE Bus STARTING 2024-05-20T22:49:51.076 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: [20/May/2024:22:49:50] ENGINE Serving on https://172.21.15.138:7150 2024-05-20T22:49:51.076 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: [20/May/2024:22:49:50] ENGINE Serving on http://172.21.15.138:8765 2024-05-20T22:49:51.076 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:50 smithi055 ceph-mon[24757]: [20/May/2024:22:49:50] ENGINE Bus STARTED 2024-05-20T22:49:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/4275516751' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-20T22:49:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: mgrmap e33: x(active, starting, since 0.0406853s) 2024-05-20T22:49:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:49:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:49:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:49:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:49:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:49:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:49:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: Manager daemon x is now available 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:49:51.181 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:49:51.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:49:51.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T22:49:51.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: [20/May/2024:22:49:50] ENGINE Bus STARTING 2024-05-20T22:49:51.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: [20/May/2024:22:49:50] ENGINE Serving on https://172.21.15.138:7150 2024-05-20T22:49:51.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: [20/May/2024:22:49:50] ENGINE Serving on http://172.21.15.138:8765 2024-05-20T22:49:51.182 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:50 smithi138 ceph-mon[25673]: [20/May/2024:22:49:50] ENGINE Bus STARTED 2024-05-20T22:49:51.923 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:51 smithi055 ceph-mon[29498]: mgrmap e34: x(active, since 1.0574s) 2024-05-20T22:49:51.923 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:51 smithi055 ceph-mon[29498]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:51.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:51 smithi055 ceph-mon[24757]: mgrmap e34: x(active, since 1.0574s) 2024-05-20T22:49:51.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:51 smithi055 ceph-mon[24757]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:51.970 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:51 smithi138 ceph-mon[25673]: mgrmap e34: x(active, since 1.0574s) 2024-05-20T22:49:51.970 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:51 smithi138 ceph-mon[25673]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:52.205 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:52 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:52.105+0000 7f362f471200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:49:52.465 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:52 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:52.288+0000 7f362f471200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:49:52.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:52 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:52.473+0000 7f362f471200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:49:53.049 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:52 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:52.786+0000 7f362f471200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:49:53.049 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:52 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:52.889+0000 7f362f471200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:49:53.312 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:53.111+0000 7f362f471200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:49:53.312 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:53.319+0000 7f362f471200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:49:53.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:53 smithi138 ceph-mon[25673]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:53.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:53 smithi138 ceph-mon[25673]: mgrmap e35: x(active, since 3s) 2024-05-20T22:49:53.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:53 smithi055 ceph-mon[29498]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:53.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:53 smithi055 ceph-mon[29498]: mgrmap e35: x(active, since 3s) 2024-05-20T22:49:53.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:53 smithi055 ceph-mon[24757]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:53.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:53 smithi055 ceph-mon[24757]: mgrmap e35: x(active, since 3s) 2024-05-20T22:49:54.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:53.986+0000 7f362f471200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:49:54.496 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:54 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:54.454+0000 7f362f471200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:49:54.798 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:54 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:54.561+0000 7f362f471200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:49:54.799 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:54 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:54.806+0000 7f362f471200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T22:49:55.637 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:55 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:55.374+0000 7f362f471200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:49:55.637 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:55 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:55.484+0000 7f362f471200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:49:55.893 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:55 smithi055 ceph-mon[24757]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:55.893 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:55 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:55.893 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:55 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:55.894 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:55 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:55.752+0000 7f362f471200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:49:55.894 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:55 smithi055 ceph-mon[29498]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:55.894 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:55 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:55.894 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:55 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:55 smithi138 ceph-mon[25673]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:56.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:55 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:55 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:55 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:55.901+0000 7f362f471200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:49:56.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:56 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:56.005+0000 7f362f471200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:49:56.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:56 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:56.130+0000 7f362f471200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:49:56.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:56 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:56 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:56 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:56 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:56 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:56 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:56 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:56 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:56.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:56 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:56.461+0000 7f362f471200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:49:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:56 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:56 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:56 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:56 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:57.468 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:57.126+0000 7f362f471200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:49:57.468 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:57.366+0000 7f362f471200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:49:57.468 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:49:57] ENGINE Bus STARTING 2024-05-20T22:49:57.468 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CherryPy Checker: 2024-05-20T22:49:57.468 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: The Application mounted at '' has an empty config. 2024-05-20T22:49:57.468 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:49:57.468 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:49:57] ENGINE Serving on http://:::9283 2024-05-20T22:49:57.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:49:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:49:57] ENGINE Bus STARTED 2024-05-20T22:49:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: Standby manager daemon y started 2024-05-20T22:49:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T22:49:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:49:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T22:49:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:49:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:57.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:49:57.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:49:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: Standby manager daemon y started 2024-05-20T22:49:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T22:49:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:49:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T22:49:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:49:57.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:57.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:57.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:49:57.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:57 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:49:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:49:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: Standby manager daemon y started 2024-05-20T22:49:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T22:49:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:49:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T22:49:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:49:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:49:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:57 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:49:58.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:49:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:49:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:49:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:49:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: mgrmap e36: x(active, since 7s), standbys: y 2024-05-20T22:49:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:49:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:49:58.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: mgrmap e36: x(active, since 7s), standbys: y 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:49:58.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:58 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:49:59.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='client.15279 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:49:59.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:49:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:49:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:49:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:49:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: mgrmap e36: x(active, since 7s), standbys: y 2024-05-20T22:49:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:49:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:49:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:49:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:49:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:58 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:49:59.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[24757]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[24757]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[24757]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[29498]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:49:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:49:59.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:49:59.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:49:59 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:50:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:59 smithi138 ceph-mon[25673]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:50:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:59 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:50:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:59 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:50:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:59 smithi138 ceph-mon[25673]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:50:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:59 smithi138 ceph-mon[25673]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:50:00.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:59 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:50:00.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:49:59 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:50:00.849 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:00 smithi055 ceph-mon[24757]: overall HEALTH_OK 2024-05-20T22:50:00.850 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:00 smithi055 ceph-mon[29498]: overall HEALTH_OK 2024-05-20T22:50:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:00 smithi138 ceph-mon[25673]: overall HEALTH_OK 2024-05-20T22:50:02.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:01 smithi138 ceph-mon[25673]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:50:02.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:50:01.613+0000 7f7b5eed7700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:50:02.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Creating ceph-iscsi config... 2024-05-20T22:50:02.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:50:02.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:50:02.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:50:02.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:02.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:02.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Traceback (most recent call last): 2024-05-20T22:50:02.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:50:02.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return _run_code(code, main_globals, None, 2024-05-20T22:50:02.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:50:02.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: exec(code, run_globals) 2024-05-20T22:50:02.108 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:50:02.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:50:02.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:50:02.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:50:02.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:50:02.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:50:02.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:50:02.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:02.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:02.109 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Traceback (most recent call last): 2024-05-20T22:50:02.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:50:02.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:50:02.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:50:02.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:50:02.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:50:02.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:50:02.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:50:02.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return future.result(timeout) 2024-05-20T22:50:02.110 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return self.__get_result() 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: raise self._exception 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: use_current_daemon_image=reconfig, 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Creating ceph-iscsi config... 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:50:02.111 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: Traceback (most recent call last): 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: return _run_code(code, main_globals, None, 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: exec(code, run_globals) 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:50:02.112 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:50:02.113 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:50:02.113 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:50:02.113 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:50:02.113 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:02.113 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:02.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:01 smithi055 ceph-mon[29498]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:50:02.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:01 smithi055 ceph-mon[24757]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:02 smithi138 ceph-mon[25673]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:50:03.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:50:03.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: return future.result(timeout) 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.__get_result() 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: raise self._exception 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:50:03.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:50:03.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:50:03.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:50:03.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:50:03.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:50:03.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:50:03.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:50:03.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:02 smithi138 ceph-mon[25673]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:50:03.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:02 smithi055 ceph-mon[29498]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:50:03.234 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:50:03.235 INFO:journalctl@ceph.mon.c.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: return future.result(timeout) 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.__get_result() 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: raise self._exception 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:50:03.236 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:50:03.237 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:02 smithi055 ceph-mon[24757]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:50:03.238 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:50:03.239 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: return future.result(timeout) 2024-05-20T22:50:03.240 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.__get_result() 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: raise self._exception 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.241 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:50:03.242 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.243 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:02 smithi055 ceph-mon[24757]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:50:03.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:50:03.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:50:03.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:50:03.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:50:03.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:50:03.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:02 smithi055 ceph-mon[29498]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:50:03.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:03 smithi138 ceph-mon[25673]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:50:03.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:03 smithi138 ceph-mon[25673]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:50:04.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:03 smithi055 ceph-mon[29498]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:50:04.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:03 smithi055 ceph-mon[29498]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:50:04.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:03 smithi055 ceph-mon[24757]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:50:04.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:03 smithi055 ceph-mon[24757]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:50:05.173 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 systemd[1]: Stopping Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:50:05.173 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 bash[57270]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:50:05.457 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.181Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-20T22:50:05.457 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.181Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-20T22:50:05.457 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.181Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-20T22:50:05.457 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.181Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-20T22:50:05.457 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.181Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-20T22:50:05.457 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.181Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-20T22:50:05.457 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.181Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-20T22:50:05.457 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.181Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-20T22:50:05.458 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.181Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-20T22:50:05.458 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.182Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-20T22:50:05.458 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.182Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-20T22:50:05.458 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[54351]: ts=2024-05-20T22:50:05.182Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-20T22:50:05.458 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 podman[57282]: 2024-05-20 22:50:05.191524414 +0000 UTC m=+0.108092374 container died 9aaf823c00efb8e4b087aae35d597010a7d0d991d0811574e053870af7f3b71a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:50:05.782 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 podman[57282]: 2024-05-20 22:50:05.563853031 +0000 UTC m=+0.480420983 container cleanup 9aaf823c00efb8e4b087aae35d597010a7d0d991d0811574e053870af7f3b71a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:50:05.782 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 bash[57282]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a 2024-05-20T22:50:05.783 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 bash[57310]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:50:05.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:05 smithi055 ceph-mon[24757]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:50:05.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:05 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:05.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:05 smithi055 ceph-mon[29498]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:50:05.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:05 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:06.085 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:05 smithi138 ceph-mon[25673]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:50:06.085 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:05 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:06.085 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 podman[57294]: 2024-05-20 22:50:05.857067255 +0000 UTC m=+0.663767053 container remove 9aaf823c00efb8e4b087aae35d597010a7d0d991d0811574e053870af7f3b71a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:50:06.085 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-20T22:50:06.085 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 systemd[1]: Stopped Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:50:06.085 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Consumed 1.388s CPU time. 2024-05-20T22:50:06.086 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:05 smithi138 systemd[1]: Starting Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:50:06.354 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 podman[57374]: 2024-05-20 22:50:05.996941766 +0000 UTC m=+0.017996067 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-20T22:50:06.354 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 podman[57374]: 2024-05-20 22:50:06.124645031 +0000 UTC m=+0.145699327 container create f6e8677f54c8a49d75f7c7d12428faf927732f549103751626b87b75ffbf9381 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:50:06.354 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 podman[57374]: 2024-05-20 22:50:06.335103869 +0000 UTC m=+0.356158171 container init f6e8677f54c8a49d75f7c7d12428faf927732f549103751626b87b75ffbf9381 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:50:06.354 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 podman[57374]: 2024-05-20 22:50:06.337845894 +0000 UTC m=+0.358900197 container start f6e8677f54c8a49d75f7c7d12428faf927732f549103751626b87b75ffbf9381 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:50:06.606 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.362Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.362Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.362Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.362Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi138 (none))" 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.362Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.362Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.364Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.365Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.365Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.367Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.367Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.509µs 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.367Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-20T22:50:06.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 bash[57374]: f6e8677f54c8a49d75f7c7d12428faf927732f549103751626b87b75ffbf9381 2024-05-20T22:50:06.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 systemd[1]: Started Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:50:06.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.567Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-05-20T22:50:07.091 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:06 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:06.826Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-05-20T22:50:07.091 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:07.049Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-05-20T22:50:07.091 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:07.096Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-05-20T22:50:07.091 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:07.096Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-05-20T22:50:07.091 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:07.096Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=57.43µs wal_replay_duration=729.693269ms total_replay_duration=729.765663ms 2024-05-20T22:50:07.356 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:07.101Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-20T22:50:07.356 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:07.101Z caller=main.go:947 level=info msg="TSDB started" 2024-05-20T22:50:07.356 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:07.101Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-20T22:50:07.357 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:07.127Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.742072ms db_storage=840ns remote_storage=1.233µs web_handler=360ns query_engine=840ns scrape=609.778µs scrape_sd=95.509µs notify=10.106µs notify_sd=13.222µs rules=24.593578ms 2024-05-20T22:50:07.357 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:50:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:50:07.127Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-20T22:50:07.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[24757]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:50:07.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[29498]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:50:07.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:07 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:50:07.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:07 smithi138 ceph-mon[25673]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:50:07.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:07 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:07.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:07 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:07.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:07 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:50:07.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:07 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:50:07.819 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:07 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:50:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:09 smithi055 ceph-mon[24757]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:50:09.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:09 smithi055 ceph-mon[29498]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:50:09.765 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:09 smithi138 ceph-mon[25673]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:50:11.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[24757]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:50:11.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:11.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:11.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:50:11.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:50:11.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:11.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[29498]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:50:11.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:11 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:50:12.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:11 smithi138 ceph-mon[25673]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:50:12.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:11 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:12.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:11 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:12.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:11 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:50:12.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:11 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:50:12.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:11 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:12.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:11 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:50:12.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:11 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:50:12.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:11 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:50:12.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:12 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:50:12.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:12 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:50:12.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:12 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:50:12.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:12 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:50:13.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:12 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:50:13.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:12 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:50:13.902 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:13 smithi055 ceph-mon[24757]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:50:13.902 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:13 smithi055 ceph-mon[29498]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:50:14.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:13 smithi138 ceph-mon[25673]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:50:15.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[24757]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:50:15.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2414956743' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[29498]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:50:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:50:15.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:15 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2414956743' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:50:16.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:15 smithi138 ceph-mon[25673]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:50:16.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:15 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:16.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:15 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:16.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:15 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:50:16.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:15 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:50:16.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:15 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:50:16.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:15 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2414956743' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:50:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:16 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:50:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:16 smithi055 ceph-mon[29498]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:50:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:16 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1807732386' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2871096746"}]: dispatch 2024-05-20T22:50:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:16 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2871096746"}]: dispatch 2024-05-20T22:50:16.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:16 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:50:16.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:16 smithi055 ceph-mon[24757]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:50:16.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:16 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1807732386' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2871096746"}]: dispatch 2024-05-20T22:50:16.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:16 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2871096746"}]: dispatch 2024-05-20T22:50:17.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:16 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:50:17.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:16 smithi138 ceph-mon[25673]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:50:17.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:16 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1807732386' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2871096746"}]: dispatch 2024-05-20T22:50:17.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:16 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2871096746"}]: dispatch 2024-05-20T22:50:17.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[24757]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:50:17.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2871096746"}]': finished 2024-05-20T22:50:17.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[24757]: osdmap e107: 8 total, 8 up, 8 in 2024-05-20T22:50:17.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/326654757' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/4175557291"}]: dispatch 2024-05-20T22:50:17.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/4175557291"}]: dispatch 2024-05-20T22:50:17.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[29498]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:50:17.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2871096746"}]': finished 2024-05-20T22:50:17.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[29498]: osdmap e107: 8 total, 8 up, 8 in 2024-05-20T22:50:17.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/326654757' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/4175557291"}]: dispatch 2024-05-20T22:50:17.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:17 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/4175557291"}]: dispatch 2024-05-20T22:50:18.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:17 smithi138 ceph-mon[25673]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:50:18.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:17 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2871096746"}]': finished 2024-05-20T22:50:18.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:17 smithi138 ceph-mon[25673]: osdmap e107: 8 total, 8 up, 8 in 2024-05-20T22:50:18.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:17 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/326654757' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/4175557291"}]: dispatch 2024-05-20T22:50:18.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:17 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/4175557291"}]: dispatch 2024-05-20T22:50:19.087 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:18 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/4175557291"}]': finished 2024-05-20T22:50:19.087 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:18 smithi055 ceph-mon[29498]: osdmap e108: 8 total, 8 up, 8 in 2024-05-20T22:50:19.087 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:18 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3450834484' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/4175557291"}]: dispatch 2024-05-20T22:50:19.087 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:18 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/4175557291"}]: dispatch 2024-05-20T22:50:19.088 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:18 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/4175557291"}]': finished 2024-05-20T22:50:19.088 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:18 smithi055 ceph-mon[24757]: osdmap e108: 8 total, 8 up, 8 in 2024-05-20T22:50:19.088 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:18 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3450834484' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/4175557291"}]: dispatch 2024-05-20T22:50:19.088 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:18 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/4175557291"}]: dispatch 2024-05-20T22:50:19.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:18 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/4175557291"}]': finished 2024-05-20T22:50:19.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:18 smithi138 ceph-mon[25673]: osdmap e108: 8 total, 8 up, 8 in 2024-05-20T22:50:19.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:18 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3450834484' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/4175557291"}]: dispatch 2024-05-20T22:50:19.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:18 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/4175557291"}]: dispatch 2024-05-20T22:50:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:19 smithi138 ceph-mon[25673]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-05-20T22:50:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:19 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/4175557291"}]': finished 2024-05-20T22:50:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:19 smithi138 ceph-mon[25673]: osdmap e109: 8 total, 8 up, 8 in 2024-05-20T22:50:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:19 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2109912434' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/4157946518"}]: dispatch 2024-05-20T22:50:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:19 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:19 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:20.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:19 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:50:20.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:19 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:50:20.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:19 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:20.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[29498]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-05-20T22:50:20.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/4175557291"}]': finished 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[29498]: osdmap e109: 8 total, 8 up, 8 in 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2109912434' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/4157946518"}]: dispatch 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[24757]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/4175557291"}]': finished 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[24757]: osdmap e109: 8 total, 8 up, 8 in 2024-05-20T22:50:20.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2109912434' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/4157946518"}]: dispatch 2024-05-20T22:50:20.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:20.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:20.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:50:20.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:50:20.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:19 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:50:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:20 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2109912434' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/4157946518"}]': finished 2024-05-20T22:50:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:20 smithi138 ceph-mon[25673]: osdmap e110: 8 total, 8 up, 8 in 2024-05-20T22:50:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:20 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:20 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2232487789' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2944137534"}]: dispatch 2024-05-20T22:50:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:20 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2944137534"}]: dispatch 2024-05-20T22:50:21.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2109912434' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/4157946518"}]': finished 2024-05-20T22:50:21.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[24757]: osdmap e110: 8 total, 8 up, 8 in 2024-05-20T22:50:21.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:21.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2232487789' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2944137534"}]: dispatch 2024-05-20T22:50:21.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2944137534"}]: dispatch 2024-05-20T22:50:21.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2109912434' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/4157946518"}]': finished 2024-05-20T22:50:21.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[29498]: osdmap e110: 8 total, 8 up, 8 in 2024-05-20T22:50:21.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:21.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2232487789' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2944137534"}]: dispatch 2024-05-20T22:50:21.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:20 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2944137534"}]: dispatch 2024-05-20T22:50:21.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:50:21] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-20T22:50:22.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:21 smithi138 ceph-mon[25673]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:50:22.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:21 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2944137534"}]': finished 2024-05-20T22:50:22.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:21 smithi138 ceph-mon[25673]: osdmap e111: 8 total, 8 up, 8 in 2024-05-20T22:50:22.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:21 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3741494196' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3994718584"}]: dispatch 2024-05-20T22:50:22.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:21 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3994718584"}]: dispatch 2024-05-20T22:50:22.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[24757]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:50:22.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2944137534"}]': finished 2024-05-20T22:50:22.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[24757]: osdmap e111: 8 total, 8 up, 8 in 2024-05-20T22:50:22.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3741494196' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3994718584"}]: dispatch 2024-05-20T22:50:22.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3994718584"}]: dispatch 2024-05-20T22:50:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[29498]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:50:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2944137534"}]': finished 2024-05-20T22:50:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[29498]: osdmap e111: 8 total, 8 up, 8 in 2024-05-20T22:50:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3741494196' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3994718584"}]: dispatch 2024-05-20T22:50:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:21 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3994718584"}]: dispatch 2024-05-20T22:50:23.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:22 smithi138 ceph-mon[25673]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3994718584"}]': finished 2024-05-20T22:50:23.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:22 smithi138 ceph-mon[25673]: osdmap e112: 8 total, 8 up, 8 in 2024-05-20T22:50:23.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:22 smithi138 ceph-mon[25673]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:23.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:22 smithi055 ceph-mon[29498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3994718584"}]': finished 2024-05-20T22:50:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:22 smithi055 ceph-mon[29498]: osdmap e112: 8 total, 8 up, 8 in 2024-05-20T22:50:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:22 smithi055 ceph-mon[29498]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:23.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:22 smithi055 ceph-mon[24757]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/3994718584"}]': finished 2024-05-20T22:50:23.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:22 smithi055 ceph-mon[24757]: osdmap e112: 8 total, 8 up, 8 in 2024-05-20T22:50:23.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:22 smithi055 ceph-mon[24757]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:25.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:24 smithi055 ceph-mon[24757]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 0 op/s 2024-05-20T22:50:25.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:24 smithi055 ceph-mon[29498]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 0 op/s 2024-05-20T22:50:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:24 smithi138 ceph-mon[25673]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 0 op/s 2024-05-20T22:50:26.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:25 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:26.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:25 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:25 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:27.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:26 smithi055 ceph-mon[24757]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-05-20T22:50:27.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:26 smithi055 ceph-mon[29498]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-05-20T22:50:27.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:26 smithi138 ceph-mon[25673]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-05-20T22:50:29.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:29 smithi055 ceph-mon[24757]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:29.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:29 smithi055 ceph-mon[29498]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:29.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:29 smithi138 ceph-mon[25673]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:31.472 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:50:31] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-20T22:50:31.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:31 smithi055 ceph-mon[24757]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T22:50:31.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:31 smithi055 ceph-mon[29498]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T22:50:31.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:31 smithi138 ceph-mon[25673]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T22:50:33.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:33 smithi055 ceph-mon[24757]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1020 B/s rd, 0 op/s 2024-05-20T22:50:33.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:33 smithi055 ceph-mon[29498]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1020 B/s rd, 0 op/s 2024-05-20T22:50:33.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:33 smithi138 ceph-mon[25673]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1020 B/s rd, 0 op/s 2024-05-20T22:50:35.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:35 smithi055 ceph-mon[24757]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:35.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:35 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:35.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:35 smithi055 ceph-mon[29498]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:35.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:35 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:35.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:35 smithi138 ceph-mon[25673]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:35.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:35 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:36 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:36.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:36 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:36.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:36 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:37.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:37 smithi138 ceph-mon[25673]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:37.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:37 smithi055 ceph-mon[24757]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:37.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:37 smithi055 ceph-mon[29498]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:39.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:39 smithi138 ceph-mon[25673]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:39.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:39 smithi055 ceph-mon[24757]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:39 smithi055 ceph-mon[29498]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:41.510 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:50:41] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-05-20T22:50:41.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:41 smithi138 ceph-mon[25673]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:41 smithi055 ceph-mon[24757]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:41.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:41 smithi055 ceph-mon[29498]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:43.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:43 smithi138 ceph-mon[25673]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:43.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:43 smithi055 ceph-mon[24757]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:43.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:43 smithi055 ceph-mon[29498]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:45.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:45 smithi055 ceph-mon[24757]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:45.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:45 smithi055 ceph-mon[29498]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:45.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:45 smithi138 ceph-mon[25673]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:46.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:46 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:46.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:46 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:46.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:46 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:47.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:47 smithi138 ceph-mon[25673]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:47.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:47 smithi055 ceph-mon[24757]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:47.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:47 smithi055 ceph-mon[29498]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:49.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:49 smithi138 ceph-mon[25673]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:49.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:49 smithi055 ceph-mon[24757]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:49.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:49 smithi055 ceph-mon[29498]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:50.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:50:51.580 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:50:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:50:51] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-05-20T22:50:51.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:51 smithi138 ceph-mon[25673]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:51 smithi055 ceph-mon[24757]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:51.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:51 smithi055 ceph-mon[29498]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:53.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:53 smithi138 ceph-mon[25673]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:53.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:53 smithi055 ceph-mon[24757]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:53.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:53 smithi055 ceph-mon[29498]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:55.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:55 smithi055 ceph-mon[24757]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:55.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:55 smithi055 ceph-mon[29498]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:56.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:55 smithi138 ceph-mon[25673]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:56 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:56.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:56 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:56 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:50:57.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:57 smithi055 ceph-mon[24757]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:57 smithi055 ceph-mon[29498]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:57 smithi138 ceph-mon[25673]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:50:59.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:50:59 smithi055 ceph-mon[24757]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:50:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:50:59 smithi055 ceph-mon[29498]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:50:59 smithi138 ceph-mon[25673]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:01.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:51:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:51:01] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-05-20T22:51:01.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:01 smithi055 ceph-mon[24757]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:01.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:01 smithi055 ceph-mon[29498]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:02.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:01 smithi138 ceph-mon[25673]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:03.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:03 smithi055 ceph-mon[24757]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:03.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:03 smithi055 ceph-mon[29498]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:04.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:03 smithi138 ceph-mon[25673]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:05.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:05 smithi055 ceph-mon[29498]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:51:05.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:05 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:05.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:05 smithi055 ceph-mon[24757]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:51:05.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:05 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:06.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:05 smithi138 ceph-mon[25673]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T22:51:06.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:05 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:06 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:06 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:07.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:06 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:07.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:07 smithi055 ceph-mon[24757]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:07.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:07 smithi055 ceph-mon[29498]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:08.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:07 smithi138 ceph-mon[25673]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:09.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:09 smithi055 ceph-mon[24757]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:09.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:09 smithi055 ceph-mon[29498]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:10.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:09 smithi138 ceph-mon[25673]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:11.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:51:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:51:11] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-05-20T22:51:11.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:11 smithi055 ceph-mon[24757]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:11.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:11 smithi055 ceph-mon[29498]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:12.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:11 smithi138 ceph-mon[25673]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:14.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:13 smithi138 ceph-mon[25673]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:14.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:13 smithi055 ceph-mon[24757]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:14.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:13 smithi055 ceph-mon[29498]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:16.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:15 smithi138 ceph-mon[25673]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:16.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:15 smithi055 ceph-mon[24757]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:16.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:15 smithi055 ceph-mon[29498]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:17.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:16 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:17.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:16 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:17.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:16 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:18.076 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:17 smithi138 ceph-mon[25673]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:18.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:17 smithi055 ceph-mon[24757]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:18.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:17 smithi055 ceph-mon[29498]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:19 smithi138 ceph-mon[25673]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:19 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:51:20.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:19 smithi055 ceph-mon[24757]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:20.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:19 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:51:20.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:19 smithi055 ceph-mon[29498]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:20.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:19 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:51:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:20 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:20 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:51:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:20 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:51:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:20 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:51:21.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:20 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:21.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:20 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:51:21.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:20 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:51:21.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:20 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:51:21.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:20 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:21.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:20 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:51:21.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:20 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:51:21.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:20 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:51:21.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:51:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:51:21] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-05-20T22:51:22.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:21 smithi138 ceph-mon[25673]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:22.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:21 smithi055 ceph-mon[24757]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:21 smithi055 ceph-mon[29498]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:24.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:23 smithi138 ceph-mon[25673]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:24.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:23 smithi055 ceph-mon[24757]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:24.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:23 smithi055 ceph-mon[29498]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:26.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:25 smithi055 ceph-mon[24757]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:26.003 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:25 smithi055 ceph-mon[29498]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:26.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:25 smithi138 ceph-mon[25673]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:27.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:26 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:27.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:26 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:27.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:26 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:27 smithi138 ceph-mon[25673]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:28.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:27 smithi055 ceph-mon[24757]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:28.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:27 smithi055 ceph-mon[29498]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:30.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:29 smithi138 ceph-mon[25673]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:30.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:29 smithi055 ceph-mon[24757]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:30.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:29 smithi055 ceph-mon[29498]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:31.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:51:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:51:31] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-05-20T22:51:32.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:31 smithi138 ceph-mon[25673]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:31 smithi055 ceph-mon[24757]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:32.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:31 smithi055 ceph-mon[29498]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:33 smithi138 ceph-mon[25673]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:34.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:33 smithi055 ceph-mon[24757]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:34.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:33 smithi055 ceph-mon[29498]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:35.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:34 smithi055 ceph-mon[24757]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:34 smithi055 ceph-mon[29498]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:35.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:34 smithi138 ceph-mon[25673]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:36.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:35 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:36.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:35 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:36.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:35 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:36.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:35 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:36.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:35 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:36.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:35 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:37.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:36 smithi055 ceph-mon[24757]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:36 smithi055 ceph-mon[29498]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:36 smithi138 ceph-mon[25673]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:39.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:39 smithi055 ceph-mon[24757]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:39.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:39 smithi055 ceph-mon[29498]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:39.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:39 smithi138 ceph-mon[25673]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:41.498 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:51:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:51:41] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-05-20T22:51:41.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:41 smithi138 ceph-mon[25673]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:41 smithi055 ceph-mon[24757]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:41.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:41 smithi055 ceph-mon[29498]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:43.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:43 smithi138 ceph-mon[25673]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:43.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:43 smithi055 ceph-mon[24757]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:43.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:43 smithi055 ceph-mon[29498]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:45.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:45 smithi055 ceph-mon[24757]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:45.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:45 smithi055 ceph-mon[29498]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:45.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:45 smithi138 ceph-mon[25673]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:46.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:46 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:46.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:46 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:46.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:46 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:47.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:47 smithi138 ceph-mon[25673]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:47.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:47 smithi055 ceph-mon[24757]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:47.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:47 smithi055 ceph-mon[29498]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:49.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:49 smithi138 ceph-mon[25673]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:49.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:49 smithi055 ceph-mon[24757]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:49.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:49 smithi055 ceph-mon[29498]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:50.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:51:51.558 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:51:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:51:51] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-05-20T22:51:51.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:51 smithi138 ceph-mon[25673]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:51.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:51 smithi055 ceph-mon[24757]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:51.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:51 smithi055 ceph-mon[29498]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:53.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:53 smithi138 ceph-mon[25673]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:53.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:53 smithi055 ceph-mon[24757]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:53.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:53 smithi055 ceph-mon[29498]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:55.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:55 smithi055 ceph-mon[24757]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:55.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:55 smithi055 ceph-mon[29498]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:55.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:55 smithi138 ceph-mon[25673]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:56.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:56 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:56 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:56.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:56 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:51:57.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:57 smithi138 ceph-mon[25673]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:57.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:57 smithi055 ceph-mon[24757]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:57 smithi055 ceph-mon[29498]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:51:59.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:51:59 smithi138 ceph-mon[25673]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:59.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:51:59 smithi055 ceph-mon[24757]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:51:59.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:51:59 smithi055 ceph-mon[29498]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:01.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:52:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:52:01] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-05-20T22:52:01.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:01 smithi055 ceph-mon[24757]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:01.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:01 smithi055 ceph-mon[29498]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:02.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:01 smithi138 ceph-mon[25673]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:03.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:03 smithi055 ceph-mon[24757]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:03.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:03 smithi055 ceph-mon[29498]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:04.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:03 smithi138 ceph-mon[25673]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:05.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:05 smithi055 ceph-mon[24757]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:05.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:05 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:05.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:05 smithi055 ceph-mon[29498]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:05.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:05 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:06.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:05 smithi138 ceph-mon[25673]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:06.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:05 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:06 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:06 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:07.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:06 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:07.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:07 smithi055 ceph-mon[24757]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:07.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:07 smithi055 ceph-mon[29498]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:08.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:07 smithi138 ceph-mon[25673]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:09.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:09 smithi055 ceph-mon[24757]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:09.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:09 smithi055 ceph-mon[29498]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:10.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:09 smithi138 ceph-mon[25673]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:11.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:52:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:52:11] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-05-20T22:52:11.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:11 smithi055 ceph-mon[24757]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:11.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:11 smithi055 ceph-mon[29498]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:12.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:11 smithi138 ceph-mon[25673]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:13.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:13 smithi055 ceph-mon[24757]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:13.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:13 smithi055 ceph-mon[29498]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:14.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:13 smithi138 ceph-mon[25673]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:15.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:15 smithi055 ceph-mon[29498]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:15.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:15 smithi055 ceph-mon[24757]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:16.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:15 smithi138 ceph-mon[25673]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:16.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:16 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:16 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:17.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:16 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:17.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:17 smithi055 ceph-mon[24757]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:17.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:17 smithi055 ceph-mon[29498]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:18.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:17 smithi138 ceph-mon[25673]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:19.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:19 smithi055 ceph-mon[24757]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:19.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:19 smithi055 ceph-mon[29498]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:19 smithi138 ceph-mon[25673]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:20.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:20 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:20.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:20 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:52:20.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:20 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:20.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:20 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:52:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:20 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:20 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:52:21.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:52:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:52:21] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-05-20T22:52:21.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:21 smithi055 ceph-mon[24757]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:21.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:21 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:52:21.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:21 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:52:21.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:21 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:52:21.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:21 smithi055 ceph-mon[29498]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:21.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:21 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:52:21.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:21 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:52:21.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:21 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:52:22.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:21 smithi138 ceph-mon[25673]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:22.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:21 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:52:22.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:21 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:52:22.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:21 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:52:23.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:23 smithi055 ceph-mon[24757]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:23.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:23 smithi055 ceph-mon[29498]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:24.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:23 smithi138 ceph-mon[25673]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:25.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:25 smithi055 ceph-mon[24757]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:25.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:25 smithi055 ceph-mon[29498]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:26.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:25 smithi138 ceph-mon[25673]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:27.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:26 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:27.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:26 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:27.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:26 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:28.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:27 smithi138 ceph-mon[25673]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:28.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:27 smithi055 ceph-mon[24757]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:28.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:27 smithi055 ceph-mon[29498]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:30.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:29 smithi138 ceph-mon[25673]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:30.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:29 smithi055 ceph-mon[24757]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:30.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:29 smithi055 ceph-mon[29498]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:31.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:52:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:52:31] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-05-20T22:52:32.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:31 smithi138 ceph-mon[25673]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:31 smithi055 ceph-mon[24757]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:32.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:31 smithi055 ceph-mon[29498]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:33 smithi138 ceph-mon[25673]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:34.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:33 smithi055 ceph-mon[24757]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:34.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:33 smithi055 ceph-mon[29498]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:36.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:35 smithi138 ceph-mon[25673]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:36.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:35 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:36.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:35 smithi055 ceph-mon[24757]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:36.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:35 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:36.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:35 smithi055 ceph-mon[29498]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:36.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:35 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:37.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:36 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:37.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:36 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:36 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:38.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:37 smithi138 ceph-mon[25673]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:38.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:37 smithi055 ceph-mon[24757]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:38.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:37 smithi055 ceph-mon[29498]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:40.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:39 smithi138 ceph-mon[25673]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:40.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:39 smithi055 ceph-mon[24757]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:40.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:39 smithi055 ceph-mon[29498]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:41.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:52:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:52:41] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-05-20T22:52:42.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:41 smithi138 ceph-mon[25673]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:42.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:41 smithi055 ceph-mon[24757]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:42.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:41 smithi055 ceph-mon[29498]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:44.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:43 smithi138 ceph-mon[25673]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:44.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:43 smithi055 ceph-mon[24757]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:44.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:43 smithi055 ceph-mon[29498]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:46.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:45 smithi138 ceph-mon[25673]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:46.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:45 smithi055 ceph-mon[24757]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:46.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:45 smithi055 ceph-mon[29498]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:47.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:46 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:47.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:46 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:47.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:46 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:48.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:47 smithi138 ceph-mon[25673]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:48.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:47 smithi055 ceph-mon[24757]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:47 smithi055 ceph-mon[29498]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:50.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:49 smithi055 ceph-mon[29498]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:50.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:49 smithi055 ceph-mon[24757]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:50.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:49 smithi138 ceph-mon[25673]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:51.207 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:50 smithi138 ceph-mon[25673]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:51.207 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:50 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:51.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:50 smithi055 ceph-mon[24757]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:51.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:50 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:51.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:50 smithi055 ceph-mon[29498]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:51.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:50 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:52:51.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:52:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:52:51] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-20T22:52:53.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:53 smithi055 ceph-mon[24757]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:53.668 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:53 smithi055 ceph-mon[29498]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:53.698 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T22:52:53.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:53 smithi138 ceph-mon[25673]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:55.650 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (12m) 2m ago 20m 22.5M - 0.23.0 ba2b418f427c a0f540c0ca04 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (12m) 2m ago 20m 50.0M - 8.3.5 dad864ee21e9 179b97bd7400 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (2m) 2m ago 19m 60.8M - 3.5 e1d6a67b021e e51a1e477fc3 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283 running (14m) 2m ago 24m 498M - 19.0.0-2695-g8a373da3 86045c3f4e75 0639e81ef3c8 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (9m) 2m ago 26m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (26m) 2m ago 26m 70.8M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (25m) 2m ago 25m 54.3M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (25m) 2m ago 25m 54.1M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (12m) 2m ago 21m 20.8M - 1.3.1 1dbe0e931976 69655ac4483b 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (12m) 2m ago 21m 18.8M - 1.3.1 1dbe0e931976 c17852f7fcd9 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (24m) 2m ago 24m 54.7M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:52:55.651 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (24m) 2m ago 24m 54.2M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:52:55.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (23m) 2m ago 23m 52.7M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:52:55.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (23m) 2m ago 23m 52.0M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:52:55.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (23m) 2m ago 23m 52.8M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:52:55.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (22m) 2m ago 22m 51.3M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:52:55.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (22m) 2m ago 22m 50.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:52:55.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (21m) 2m ago 21m 52.7M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:52:55.652 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (2m) 2m ago 21m 42.1M - 2.33.4 514e6a882f6e f6e8677f54c8 2024-05-20T22:52:55.652 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (19m) 2m ago 19m 74.2M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:52:55.652 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (19m) 2m ago 19m 77.7M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:52:55.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:55 smithi055 ceph-mon[24757]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:55.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:55 smithi055 ceph-mon[29498]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:55.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:55 smithi138 ceph-mon[25673]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:56.268 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ls' 2024-05-20T22:52:56.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:56 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:56.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:56 smithi055 ceph-mon[24757]: from='client.25312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:52:56.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:56 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:56.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:56 smithi055 ceph-mon[29498]: from='client.25312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:52:56.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:56 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:52:56.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:56 smithi138 ceph-mon[25673]: from='client.25312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:52:57.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:57 smithi055 ceph-mon[24757]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:57.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:57 smithi055 ceph-mon[29498]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:57.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:57 smithi138 ceph-mon[25673]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:52:58.122 INFO:teuthology.orchestra.run.smithi055.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-20T22:52:58.123 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager ?:9093,9094 1/1 2m ago 21m smithi055=a;count:1 2024-05-20T22:52:58.123 INFO:teuthology.orchestra.run.smithi055.stdout:grafana ?:3000 1/1 2m ago 21m smithi138=a;count:1 2024-05-20T22:52:58.123 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo ?:5000 1/1 2m ago 19m count:1 2024-05-20T22:52:58.123 INFO:teuthology.orchestra.run.smithi055.stdout:mgr 2/2 2m ago 25m smithi055=y;smithi138=x;count:2 2024-05-20T22:52:58.123 INFO:teuthology.orchestra.run.smithi055.stdout:mon 3/3 2m ago 25m smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi138:172.21.15.138=b;count:3 2024-05-20T22:52:58.123 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter ?:9100 2/2 2m ago 21m smithi055=a;smithi138=b;count:2 2024-05-20T22:52:58.123 INFO:teuthology.orchestra.run.smithi055.stdout:osd 8 2m ago - 2024-05-20T22:52:58.123 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus ?:9095 1/1 2m ago 21m smithi138=a;count:1 2024-05-20T22:52:58.123 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo ?:8000 2/2 2m ago 19m count:2 2024-05-20T22:52:58.729 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions' 2024-05-20T22:52:59.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:59 smithi055 ceph-mon[24757]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:59.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:52:59 smithi055 ceph-mon[24757]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:52:59.626 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:59 smithi055 ceph-mon[29498]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:59.626 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:52:59 smithi055 ceph-mon[29498]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:52:59.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:59 smithi138 ceph-mon[25673]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:52:59.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:52:59 smithi138 ceph-mon[25673]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:00.702 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:53:00.702 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:53:00.703 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:53:00.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:00.703 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:53:00.703 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:53:00.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:00.704 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:53:00.704 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:53:00.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:00.704 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:53:00.704 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:53:00.704 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:53:00.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:00.705 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:53:00.705 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:53:00.705 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:53:00.705 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:53:00.705 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:53:01.278 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-05-20T22:53:01.525 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:53:01] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-05-20T22:53:01.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:01 smithi055 ceph-mon[24757]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:01.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:01 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2318337616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:01.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:01 smithi055 ceph-mon[29498]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:01.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:01 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2318337616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:01.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:01 smithi138 ceph-mon[25673]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:01.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:01 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2318337616' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:03.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:03 smithi138 ceph-mon[25673]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:53:03.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:03 smithi055 ceph-mon[24757]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:53:03.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:03 smithi055 ceph-mon[29498]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:53:04.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:04 smithi055 ceph-mon[24757]: from='client.25330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:04.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:04 smithi055 ceph-mon[29498]: from='client.25330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:04 smithi138 ceph-mon[25673]: from='client.25330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:05.275 INFO:teuthology.orchestra.run.smithi055.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:53:05.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[24757]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:05.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:05.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:53:05.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:05.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:05.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:53:05.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:53:05.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[29498]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:05.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:05.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:53:05.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:05.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:05.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:53:05.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:05 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:53:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:05 smithi138 ceph-mon[25673]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:05 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:05 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:53:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:05 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:05 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:05 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:53:05.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:05 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:53:05.880 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-20T22:53:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:06 smithi138 ceph-mon[25673]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:53:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:06 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:53:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:06 smithi138 ceph-mon[25673]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:53:06.980 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:06 smithi055 ceph-mon[24757]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:53:06.980 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:06 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:53:06.980 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:06 smithi055 ceph-mon[24757]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:53:06.980 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:06 smithi055 ceph-mon[29498]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:53:06.980 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:06 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:53:06.980 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:06 smithi055 ceph-mon[29498]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:53:07.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:07 smithi055 ceph-mon[24757]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:53:07.644 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:07 smithi055 ceph-mon[29498]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:53:07.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:07 smithi138 ceph-mon[25673]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:53:09.026 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:53:09.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:09.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:53:09.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:53:09.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:53:09.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:09.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-20T22:53:09.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-20T22:53:09.599 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: Failing over to other MGR 2024-05-20T22:53:09.599 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-20T22:53:09.599 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-20T22:53:09.599 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[24757]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:53:09.599 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:53:09] ENGINE Bus STOPPING 2024-05-20T22:53:09.599 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:53:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T22:53:09.599 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:53:09] ENGINE Bus STOPPED 2024-05-20T22:53:09.599 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:53:09] ENGINE Bus STARTING 2024-05-20T22:53:09.599 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:53:09] ENGINE Serving on http://:::9283 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:53:09] ENGINE Bus STARTED 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: Failing over to other MGR 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-20T22:53:09.600 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-20T22:53:09.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:09 smithi055 ceph-mon[29498]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:53:09.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:09 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:09.341+0000 7f7bcc959700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-20T22:53:09.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:09 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ignoring --setuser ceph since I am not root 2024-05-20T22:53:09.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:09 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ignoring --setgroup ceph since I am not root 2024-05-20T22:53:09.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mgr[45761]: -- 172.21.15.138:0/3655555497 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5593d61565a0 con 0x5593d612ac00 2024-05-20T22:53:09.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:09 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:09.518+0000 7f1c64afc200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:53:09.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:09.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' 2024-05-20T22:53:09.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:53:09.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:53:09.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:09.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-20T22:53:09.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-20T22:53:09.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: Failing over to other MGR 2024-05-20T22:53:09.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: from='mgr.34949 172.21.15.138:0/646064473' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-20T22:53:09.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-20T22:53:09.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:09 smithi138 ceph-mon[25673]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:53:10.106 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:09 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:09.710+0000 7f1c64afc200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:53:10.106 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:09 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:09.819+0000 7f1c64afc200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:53:10.563 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:10 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:10.154+0000 7f1c64afc200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:53:10.563 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:10 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:10.254+0000 7f1c64afc200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:53:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.34949 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: mgrmap e37: y(active, starting, since 0.998637s) 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:53:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: Manager daemon y is now available 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: [20/May/2024:22:53:09] ENGINE Bus STARTING 2024-05-20T22:53:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:10 smithi138 ceph-mon[25673]: [20/May/2024:22:53:09] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:53:10.709 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (12m) 2m ago 20m 22.5M - 0.23.0 ba2b418f427c a0f540c0ca04 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (12m) 3m ago 20m 50.0M - 8.3.5 dad864ee21e9 179b97bd7400 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (2m) 2m ago 19m 60.8M - 3.5 e1d6a67b021e e51a1e477fc3 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283 running (15m) 3m ago 25m 498M - 19.0.0-2695-g8a373da3 86045c3f4e75 0639e81ef3c8 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (9m) 2m ago 26m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (26m) 2m ago 26m 70.8M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (25m) 3m ago 25m 54.3M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (25m) 2m ago 25m 54.1M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (12m) 2m ago 21m 20.8M - 1.3.1 1dbe0e931976 69655ac4483b 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (12m) 3m ago 21m 18.8M - 1.3.1 1dbe0e931976 c17852f7fcd9 2024-05-20T22:53:10.710 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (24m) 2m ago 24m 54.7M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:53:10.711 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (24m) 2m ago 24m 54.2M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:53:10.711 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (24m) 2m ago 24m 52.7M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:53:10.711 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (23m) 2m ago 23m 52.0M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:53:10.711 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (23m) 3m ago 23m 52.8M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:53:10.711 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (22m) 3m ago 22m 51.3M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:53:10.711 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (22m) 3m ago 22m 50.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:53:10.711 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (22m) 3m ago 22m 52.7M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:53:10.711 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (3m) 3m ago 21m 42.1M - 2.33.4 514e6a882f6e f6e8677f54c8 2024-05-20T22:53:10.711 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (19m) 2m ago 19m 74.2M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:53:10.712 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (19m) 3m ago 19m 77.7M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:53:10.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.34949 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-20T22:53:10.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: mgrmap e37: y(active, starting, since 0.998637s) 2024-05-20T22:53:10.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:53:10.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:53:10.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:53:10.713 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: Manager daemon y is now available 2024-05-20T22:53:10.714 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:10.714 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:10.714 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:53:10.714 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:53:10.714 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: [20/May/2024:22:53:09] ENGINE Bus STARTING 2024-05-20T22:53:10.714 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[24757]: [20/May/2024:22:53:09] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:53:10.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.34949 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-20T22:53:10.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: mgrmap e37: y(active, starting, since 0.998637s) 2024-05-20T22:53:10.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:53:10.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:53:10.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:53:10.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:53:10.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:53:10.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: Manager daemon y is now available 2024-05-20T22:53:10.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:10.722 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:10.722 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:53:10.722 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:53:10.722 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: [20/May/2024:22:53:09] ENGINE Bus STARTING 2024-05-20T22:53:10.722 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:10 smithi055 ceph-mon[29498]: [20/May/2024:22:53:09] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:53:11.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:11.131 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:53:11.131 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:53:11.131 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:53:11.131 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:11.131 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:53:11.131 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:53:11.131 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:53:11.131 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:53:11.131 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:53:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[24757]: [20/May/2024:22:53:09] ENGINE Serving on http://172.21.15.55:8765 2024-05-20T22:53:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[24757]: [20/May/2024:22:53:09] ENGINE Bus STARTED 2024-05-20T22:53:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[24757]: mgrmap e38: y(active, since 2s) 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[24757]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[24757]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[24757]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/492300631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[29498]: [20/May/2024:22:53:09] ENGINE Serving on http://172.21.15.55:8765 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[29498]: [20/May/2024:22:53:09] ENGINE Bus STARTED 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[29498]: mgrmap e38: y(active, since 2s) 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[29498]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[29498]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[29498]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:11.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:11 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/492300631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [ 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr" 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "2/2 daemons upgraded", 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "", 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:53:11.485 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:53:11.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:11.518+0000 7f1c64afc200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:53:11.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:11 smithi138 ceph-mon[25673]: [20/May/2024:22:53:09] ENGINE Serving on http://172.21.15.55:8765 2024-05-20T22:53:11.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:11 smithi138 ceph-mon[25673]: [20/May/2024:22:53:09] ENGINE Bus STARTED 2024-05-20T22:53:11.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:11 smithi138 ceph-mon[25673]: mgrmap e38: y(active, since 2s) 2024-05-20T22:53:11.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:11 smithi138 ceph-mon[25673]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:11.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:11 smithi138 ceph-mon[25673]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:11.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:11 smithi138 ceph-mon[25673]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:11.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:11 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/492300631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:12.099 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:11.698+0000 7f1c64afc200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:53:12.099 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:11.895+0000 7f1c64afc200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:53:12.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:12.206+0000 7f1c64afc200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:53:12.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:12.304+0000 7f1c64afc200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:53:12.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:12 smithi055 ceph-mon[29498]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:12.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:12 smithi055 ceph-mon[29498]: from='client.25384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:12.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:12 smithi055 ceph-mon[24757]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:12.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:12 smithi055 ceph-mon[24757]: from='client.25384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:12.718 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:12 smithi138 ceph-mon[25673]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:12.719 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:12 smithi138 ceph-mon[25673]: from='client.25384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:12.719 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:12.515+0000 7f1c64afc200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:53:12.972 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:12.726+0000 7f1c64afc200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:53:13.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:13 smithi138 ceph-mon[25673]: mgrmap e39: y(active, since 4s) 2024-05-20T22:53:13.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:13.363+0000 7f1c64afc200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:53:13.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:13 smithi055 ceph-mon[24757]: mgrmap e39: y(active, since 4s) 2024-05-20T22:53:13.677 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:13 smithi055 ceph-mon[29498]: mgrmap e39: y(active, since 4s) 2024-05-20T22:53:14.106 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:13.816+0000 7f1c64afc200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:53:14.106 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:13.915+0000 7f1c64afc200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:53:14.494 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:14.136+0000 7f1c64afc200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T22:53:14.791 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:14.699+0000 7f1c64afc200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:53:14.792 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:14.798+0000 7f1c64afc200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:53:14.792 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:14 smithi138 ceph-mon[25673]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:14.792 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:14 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.792 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:14 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.792 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:14 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.792 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:14 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.792 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:14 smithi138 ceph-mon[25673]: mgrmap e40: y(active, since 6s) 2024-05-20T22:53:14.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[24757]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:14.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[24757]: mgrmap e40: y(active, since 6s) 2024-05-20T22:53:14.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[29498]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:14.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:14.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:14 smithi055 ceph-mon[29498]: mgrmap e40: y(active, since 6s) 2024-05-20T22:53:15.309 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:15.064+0000 7f1c64afc200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:53:15.310 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:15.210+0000 7f1c64afc200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:53:15.310 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:15.317+0000 7f1c64afc200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:53:15.585 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:15.446+0000 7f1c64afc200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:53:15.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:15.770+0000 7f1c64afc200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:53:16.551 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:16.426+0000 7f1c64afc200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:53:16.551 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.551 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.551 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.551 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:53:16.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:16 smithi138 ceph-mon[25673]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:53:16.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:53:16.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[24757]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:53:16.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:16.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:16.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:53:16.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:53:16.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:53:16.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:53:16.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: from='client.35057 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:53:16.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:53:16.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:53:16.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:53:16.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:16 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:53:16.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:16.656+0000 7f1c64afc200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:53:16.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:53:16] ENGINE Bus STARTING 2024-05-20T22:53:16.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: CherryPy Checker: 2024-05-20T22:53:16.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: The Application mounted at '' has an empty config. 2024-05-20T22:53:16.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: 2024-05-20T22:53:16.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:53:16] ENGINE Serving on http://:::9283 2024-05-20T22:53:16.857 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: [20/May/2024:22:53:16] ENGINE Bus STARTED 2024-05-20T22:53:17.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:53:17.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:53:17.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: Standby manager daemon x started 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:53:17.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:17 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:53:17.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:53:17.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:53:17.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: Standby manager daemon x started 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:53:17.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:53:17.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: Standby manager daemon x started 2024-05-20T22:53:17.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:53:17.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:53:17.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:53:17.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:17 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1243130806' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:53:18.661 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:18 smithi055 ceph-mon[24757]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:53:18.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:18 smithi055 ceph-mon[24757]: mgrmap e41: y(active, since 9s), standbys: x 2024-05-20T22:53:18.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:18 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:53:18.662 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:18 smithi055 ceph-mon[29498]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:53:18.662 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:18 smithi055 ceph-mon[29498]: mgrmap e41: y(active, since 9s), standbys: x 2024-05-20T22:53:18.662 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:18 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:53:18.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:18 smithi138 ceph-mon[25673]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:53:18.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:18 smithi138 ceph-mon[25673]: mgrmap e41: y(active, since 9s), standbys: x 2024-05-20T22:53:18.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:18 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:53:20.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:53:19.883+0000 7f35b53ff700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:20.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:53:20.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:20.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:20.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:20.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:20.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:20.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:20.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return future.result(timeout) 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.__get_result() 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:53:20.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: raise self._exception 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: use_current_daemon_image=reconfig, 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:53:20.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:20.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:53:20.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:20.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:53:20.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:20.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:20.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:20.240 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:20.240 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:20.240 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:20.240 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:20.240 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.240 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.714 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:20 smithi138 ceph-mon[25673]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:53:20.714 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:20 smithi138 ceph-mon[25673]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:20.714 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:20.714 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:20.714 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:20.714 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:20.714 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.714 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.714 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.715 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: return future.result(timeout) 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.__get_result() 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: raise self._exception 2024-05-20T22:53:20.716 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.717 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:20.718 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.719 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.719 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:20 smithi138 ceph-mon[25673]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:53:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:20 smithi055 ceph-mon[24757]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:53:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:20 smithi055 ceph-mon[24757]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:53:20.735 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: return future.result(timeout) 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.__get_result() 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: raise self._exception 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:20.736 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:20.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:20 smithi055 ceph-mon[29498]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:20 smithi055 ceph-mon[29498]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:20.738 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:53:20.739 INFO:journalctl@ceph.mon.c.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: return future.result(timeout) 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.__get_result() 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: raise self._exception 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:20.740 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:20.741 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:20.742 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:20.742 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:20.742 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:20.742 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.742 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:20 smithi055 ceph-mon[24757]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:53:20.745 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:20.745 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:20.745 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:20.745 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:20.745 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:20.745 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:20 smithi055 ceph-mon[29498]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-20T22:53:21.357 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[45757]: ::ffff:172.21.15.138 - - [20/May/2024:22:53:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-20T22:53:21.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:21 smithi055 ceph-mon[24757]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:53:21.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:21 smithi055 ceph-mon[29498]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:53:21.832 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:21 smithi138 ceph-mon[25673]: Reconfiguring daemon prometheus.a on smithi138 2024-05-20T22:53:22.634 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:22 smithi138 ceph-mon[25673]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 76 op/s 2024-05-20T22:53:22.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:22 smithi055 ceph-mon[29498]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 76 op/s 2024-05-20T22:53:22.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:22 smithi055 ceph-mon[24757]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 76 op/s 2024-05-20T22:53:23.766 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 systemd[1]: Stopping Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:53:23.766 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 bash[60198]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:53:24.024 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.774Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.774Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.774Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.774Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.774Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.774Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.774Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.774Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.774Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.775Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.775Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[57390]: ts=2024-05-20T22:53:23.775Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-20T22:53:24.025 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:23 smithi138 podman[60210]: 2024-05-20 22:53:23.784490099 +0000 UTC m=+0.106686726 container died f6e8677f54c8a49d75f7c7d12428faf927732f549103751626b87b75ffbf9381 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:24.356 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 podman[60210]: 2024-05-20 22:53:24.13211271 +0000 UTC m=+0.454309353 container cleanup f6e8677f54c8a49d75f7c7d12428faf927732f549103751626b87b75ffbf9381 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:24.356 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 bash[60210]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a 2024-05-20T22:53:24.356 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 bash[60237]: Error: no container with name or ID "ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus.a" found: no such container 2024-05-20T22:53:24.709 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:24 smithi138 ceph-mon[25673]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 0 B/s wr, 70 op/s 2024-05-20T22:53:24.709 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:24.709 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 podman[60224]: 2024-05-20 22:53:24.450092732 +0000 UTC m=+0.666647108 container remove f6e8677f54c8a49d75f7c7d12428faf927732f549103751626b87b75ffbf9381 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:24.710 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-20T22:53:24.710 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 systemd[1]: Stopped Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:53:24.710 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Consumed 1.442s CPU time. 2024-05-20T22:53:24.710 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 systemd[1]: Starting Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:53:24.710 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 podman[60300]: 2024-05-20 22:53:24.583650254 +0000 UTC m=+0.018195837 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-20T22:53:24.710 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 podman[60300]: 2024-05-20 22:53:24.717948096 +0000 UTC m=+0.152493681 container create b3ca603550e1f2ae479b39f83a21923148d8b04e8914a4dee5f74031a290f199 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:24.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:24 smithi055 ceph-mon[24757]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 0 B/s wr, 70 op/s 2024-05-20T22:53:24.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:24.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:24 smithi055 ceph-mon[29498]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 0 B/s wr, 70 op/s 2024-05-20T22:53:24.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:25.012 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 podman[60300]: 2024-05-20 22:53:24.928681959 +0000 UTC m=+0.363227544 container init b3ca603550e1f2ae479b39f83a21923148d8b04e8914a4dee5f74031a290f199 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:25.012 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 podman[60300]: 2024-05-20 22:53:24.93244318 +0000 UTC m=+0.366988773 container start b3ca603550e1f2ae479b39f83a21923148d8b04e8914a4dee5f74031a290f199 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:25.012 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.957Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-20T22:53:25.012 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.957Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-20T22:53:25.012 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.957Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-20T22:53:25.012 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.957Z caller=main.go:518 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi138 (none))" 2024-05-20T22:53:25.012 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.957Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-20T22:53:25.013 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.958Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-20T22:53:25.013 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.959Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-20T22:53:25.013 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.959Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-20T22:53:25.013 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.960Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-20T22:53:25.013 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.963Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-20T22:53:25.013 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.963Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.417µs 2024-05-20T22:53:25.013 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:24.963Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-20T22:53:25.013 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 bash[60300]: b3ca603550e1f2ae479b39f83a21923148d8b04e8914a4dee5f74031a290f199 2024-05-20T22:53:25.356 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 systemd[1]: Started Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:53:25.356 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.170Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-05-20T22:53:25.657 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.396Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-05-20T22:53:25.657 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.612Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-05-20T22:53:25.657 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.665Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-05-20T22:53:25.958 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.670Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-05-20T22:53:25.958 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.670Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-05-20T22:53:25.958 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.670Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=61.821µs wal_replay_duration=707.133738ms total_replay_duration=707.216838ms 2024-05-20T22:53:25.958 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.674Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-20T22:53:25.958 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.675Z caller=main.go:947 level=info msg="TSDB started" 2024-05-20T22:53:25.958 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.675Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-20T22:53:25.958 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.694Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.168845ms db_storage=756ns remote_storage=1.246µs web_handler=362ns query_engine=687ns scrape=264.854µs scrape_sd=108.598µs notify=12.432µs notify_sd=11.082µs rules=18.344633ms 2024-05-20T22:53:25.958 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:53:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:53:25.694Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-20T22:53:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:26 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:26 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:26 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:53:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:26 smithi138 ceph-mon[25673]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:53:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:26 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:26 smithi138 ceph-mon[25673]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 0 B/s wr, 70 op/s 2024-05-20T22:53:26.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:26.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:26.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:53:26.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[24757]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:53:26.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:26.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[24757]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 0 B/s wr, 70 op/s 2024-05-20T22:53:26.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:26.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:26.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:53:26.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[29498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:53:26.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:26.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:26 smithi055 ceph-mon[29498]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 0 B/s wr, 70 op/s 2024-05-20T22:53:27.344 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:27 smithi138 ceph-mon[25673]: Upgrade: Updating mgr.x 2024-05-20T22:53:27.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:53:27.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:53:27.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:27.345 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:27 smithi138 ceph-mon[25673]: Deploying daemon mgr.x on smithi138 2024-05-20T22:53:27.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[24757]: Upgrade: Updating mgr.x 2024-05-20T22:53:27.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:53:27.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:53:27.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:27.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[24757]: Deploying daemon mgr.x on smithi138 2024-05-20T22:53:27.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[29498]: Upgrade: Updating mgr.x 2024-05-20T22:53:27.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:53:27.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:53:27.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:27.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:27 smithi055 ceph-mon[29498]: Deploying daemon mgr.x on smithi138 2024-05-20T22:53:28.812 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:28 smithi138 ceph-mon[25673]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-05-20T22:53:28.812 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:28 smithi138 systemd[1]: Stopping Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:53:28.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:28 smithi055 ceph-mon[24757]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-05-20T22:53:28.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:28 smithi055 ceph-mon[29498]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-05-20T22:53:29.106 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:28 smithi138 podman[60635]: 2024-05-20 22:53:28.819906114 +0000 UTC m=+0.168953812 container died 0639e81ef3c82f2216fc95f2a4a1adc12a9db69215512a368b85fba6b7608357 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux ) 2024-05-20T22:53:29.550 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:29 smithi138 podman[60635]: 2024-05-20 22:53:29.155500062 +0000 UTC m=+0.504547752 container cleanup 0639e81ef3c82f2216fc95f2a4a1adc12a9db69215512a368b85fba6b7608357 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514) 2024-05-20T22:53:29.550 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:29 smithi138 bash[60635]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x 2024-05-20T22:53:29.550 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:29 smithi138 podman[60648]: 2024-05-20 22:53:29.34832575 +0000 UTC m=+0.530671836 container remove 0639e81ef3c82f2216fc95f2a4a1adc12a9db69215512a368b85fba6b7608357 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-20T22:53:29.550 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:29 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-20T22:53:29.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:29 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-05-20T22:53:29.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:29 smithi138 systemd[1]: Stopped Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:53:29.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:29 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.x.service: Consumed 58.875s CPU time. 2024-05-20T22:53:29.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:29 smithi138 systemd[1]: Starting Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:53:30.252 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:29 smithi138 podman[60780]: 2024-05-20 22:53:29.876834975 +0000 UTC m=+0.017507294 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:53:30.252 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:30 smithi138 podman[60780]: 2024-05-20 22:53:30.016040676 +0000 UTC m=+0.156712976 container create 6ba160359cab3eae8a55d4e2191214ff32a05ef036e887d933d0e14c846ad80f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-05-20T22:53:30.511 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:30 smithi138 podman[60780]: 2024-05-20 22:53:30.260620006 +0000 UTC m=+0.401292321 container init 6ba160359cab3eae8a55d4e2191214ff32a05ef036e887d933d0e14c846ad80f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-20T22:53:30.511 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:30 smithi138 podman[60780]: 2024-05-20 22:53:30.263434288 +0000 UTC m=+0.404106597 container start 6ba160359cab3eae8a55d4e2191214ff32a05ef036e887d933d0e14c846ad80f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2) 2024-05-20T22:53:30.511 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:30 smithi138 ceph-mgr[60799]: -- 172.21.15.138:0/3741240920 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55dd2df9e5a0 con 0x55dd2df72c00 2024-05-20T22:53:30.511 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:30 smithi138 bash[60780]: 6ba160359cab3eae8a55d4e2191214ff32a05ef036e887d933d0e14c846ad80f 2024-05-20T22:53:30.511 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:30 smithi138 systemd[1]: Started Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:53:30.511 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:30.416+0000 7f8ea23c8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:53:30.748 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:30 smithi055 ceph-mon[24757]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-20T22:53:30.748 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:30.748 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:30.748 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:30.748 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:30 smithi055 ceph-mon[29498]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-20T22:53:30.748 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:30.748 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:30.749 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:30.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:30 smithi138 ceph-mon[25673]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-20T22:53:30.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:30.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:30.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:30.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:30.612+0000 7f8ea23c8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:53:30.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:30.721+0000 7f8ea23c8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:53:31.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:31.062+0000 7f8ea23c8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:53:31.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:31.153+0000 7f8ea23c8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:53:32.513 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:32.405+0000 7f8ea23c8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:53:32.792 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:32.612+0000 7f8ea23c8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:53:32.792 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:32 smithi138 ceph-mon[25673]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-20T22:53:32.792 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:32 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1897285730' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:53:32.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:32 smithi055 ceph-mon[29498]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-20T22:53:32.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:32 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1897285730' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:53:32.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:32 smithi055 ceph-mon[24757]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-20T22:53:32.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:32 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1897285730' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:53:33.099 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:32.799+0000 7f8ea23c8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:53:33.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:33.106+0000 7f8ea23c8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:53:33.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:33.201+0000 7f8ea23c8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:53:33.724 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:33.412+0000 7f8ea23c8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:53:33.724 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:33.634+0000 7f8ea23c8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:53:34.518 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:34.266+0000 7f8ea23c8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:53:34.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:34 smithi138 ceph-mon[25673]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-05-20T22:53:34.806 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:34.713+0000 7f8ea23c8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:53:34.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:34 smithi055 ceph-mon[24757]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-05-20T22:53:34.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:34 smithi055 ceph-mon[29498]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-05-20T22:53:35.061 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:34.813+0000 7f8ea23c8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:53:35.061 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:35.052+0000 7f8ea23c8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T22:53:35.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:35.610+0000 7f8ea23c8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:53:35.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:35.711+0000 7f8ea23c8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:53:36.209 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:35 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:36.209 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:35 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:36.209 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:35.969+0000 7f8ea23c8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:53:36.210 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:36.113+0000 7f8ea23c8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:53:36.210 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:36.216+0000 7f8ea23c8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:53:36.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:35 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:36.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:35 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:36.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:35 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:36.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:35 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:36.605 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:36.344+0000 7f8ea23c8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:53:36.944 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:36.670+0000 7f8ea23c8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:53:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:36 smithi055 ceph-mon[29498]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-05-20T22:53:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:36 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:36 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:37.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:36 smithi055 ceph-mon[24757]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-05-20T22:53:37.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:36 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:37.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:36 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:37.290 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:36 smithi138 ceph-mon[25673]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-05-20T22:53:37.291 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:36 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:37.291 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:36 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:37.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:37.298+0000 7f8ea23c8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:53:37.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:37.532+0000 7f8ea23c8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:53:37.606 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: [20/May/2024:22:53:37] ENGINE Bus STARTING 2024-05-20T22:53:37.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:53:37] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-05-20T22:53:37.991 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: CherryPy Checker: 2024-05-20T22:53:37.991 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: The Application mounted at '' has an empty config. 2024-05-20T22:53:37.991 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:53:37.991 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: [20/May/2024:22:53:37] ENGINE Serving on http://:::9283 2024-05-20T22:53:37.991 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:53:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: [20/May/2024:22:53:37] ENGINE Bus STARTED 2024-05-20T22:53:38.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:38 smithi138 ceph-mon[25673]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:53:38.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:38 smithi138 ceph-mon[25673]: Standby manager daemon x restarted 2024-05-20T22:53:38.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:38 smithi138 ceph-mon[25673]: Standby manager daemon x started 2024-05-20T22:53:38.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:38 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:53:38.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:38 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:53:38.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:38 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:53:38.755 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:38 smithi138 ceph-mon[25673]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:53:38.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[24757]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:53:38.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[24757]: Standby manager daemon x restarted 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[24757]: Standby manager daemon x started 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[24757]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[29498]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[29498]: Standby manager daemon x restarted 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[29498]: Standby manager daemon x started 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:53:38.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:53:38.778 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:38 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/1071651387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:53:39.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:39 smithi138 ceph-mon[25673]: mgrmap e42: y(active, since 30s), standbys: x 2024-05-20T22:53:39.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:39 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:39.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:39 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:39.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:39 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:39.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:39 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:53:39.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:39 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:39.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:39 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:39.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[24757]: mgrmap e42: y(active, since 30s), standbys: x 2024-05-20T22:53:39.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:39.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:39.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:39.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:53:39.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:39.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[29498]: mgrmap e42: y(active, since 30s), standbys: x 2024-05-20T22:53:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:53:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:39.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:39 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:40.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[24757]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-20T22:53:40.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:53:40.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:53:40.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:40.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:53:40.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[29498]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-20T22:53:40.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:53:40.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:53:40.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:40.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:40 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:53:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:40 smithi138 ceph-mon[25673]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-20T22:53:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:40 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:53:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:40 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:53:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:40 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:53:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:40 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:53:41.870 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (13m) 28s ago 21m 22.5M - 0.23.0 ba2b418f427c a0f540c0ca04 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (13m) 7s ago 20m 50.9M - 8.3.5 dad864ee21e9 179b97bd7400 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (3m) 28s ago 20m 53.3M - 3.5 e1d6a67b021e e51a1e477fc3 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (12s) 7s ago 25m 65.9M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (10m) 28s ago 27m 477M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (27m) 28s ago 27m 72.2M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (26m) 7s ago 26m 56.5M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (26m) 28s ago 26m 52.6M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (13m) 28s ago 21m 17.1M - 1.3.1 1dbe0e931976 69655ac4483b 2024-05-20T22:53:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (13m) 7s ago 21m 18.4M - 1.3.1 1dbe0e931976 c17852f7fcd9 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (25m) 28s ago 25m 53.9M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (25m) 28s ago 25m 54.5M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (24m) 28s ago 24m 53.2M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (24m) 28s ago 24m 52.4M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (23m) 7s ago 23m 55.4M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (23m) 7s ago 23m 52.4M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (23m) 7s ago 23m 52.6M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (22m) 7s ago 22m 53.0M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (17s) 7s ago 21m 55.3M - 2.33.4 514e6a882f6e b3ca603550e1 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (20m) 28s ago 20m 74.8M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:53:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (20m) 7s ago 20m 78.5M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:53:42.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:42 smithi138 ceph-mon[25673]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:42.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:42 smithi138 ceph-mon[25673]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:42.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:42 smithi055 ceph-mon[24757]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:42.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:42 smithi055 ceph-mon[24757]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:42.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:42 smithi055 ceph-mon[29498]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:42.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:42 smithi055 ceph-mon[29498]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:42.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:53:42.665+0000 7f35b53ff700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:42.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:53:42.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:42.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:42.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:42.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:42.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return future.result(timeout) 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:53:42.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.__get_result() 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: raise self._exception 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: use_current_daemon_image=reconfig, 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:42.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:42.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.010 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:53:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:53:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:53:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:53:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:53:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:53:43.012 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [ 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr" 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "2/2 daemons upgraded", 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading node-exporter daemons", 2024-05-20T22:53:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:53:43.377 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:53:43.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='client.25432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:43.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:43.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:43.637 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:43.637 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:43.638 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:53:43.639 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: return future.result(timeout) 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.__get_result() 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: raise self._exception 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:43.640 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:43.641 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:43.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='client.25432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:43.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:43.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:43.642 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:43.642 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:43.642 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:43.642 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:43.642 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.642 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:43.643 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: return future.result(timeout) 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.__get_result() 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: raise self._exception 2024-05-20T22:53:43.644 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:43.645 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all mgr 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all crash 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all mds 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:53:43.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all nfs 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1810425732' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all mgr 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.649 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all crash 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all mds 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all nfs 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:43 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1810425732' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='client.25432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:43.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:43.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:43.857 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:53:43.858 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: return future.result(timeout) 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.__get_result() 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: raise self._exception 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:53:43.859 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:53:43.860 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: Upgrade: Setting container_image for all mgr 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-20T22:53:43.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: Upgrade: Setting container_image for all crash 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: Upgrade: Setting container_image for all mds 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T22:53:43.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: Upgrade: Setting container_image for all nfs 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:43.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:43 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1810425732' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:53:45.014 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:44 smithi055 ceph-mon[24757]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:45.014 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:44 smithi055 ceph-mon[24757]: from='client.25450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:45.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:44 smithi055 ceph-mon[29498]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:45.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:44 smithi055 ceph-mon[29498]: from='client.25450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:45.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:44 smithi138 ceph-mon[25673]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:45.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:44 smithi138 ceph-mon[25673]: from='client.25450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:53:45.483 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:45 smithi055 systemd[1]: Stopping Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:53:45.743 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:45 smithi055 podman[76716]: 2024-05-20 22:53:45.497096297 +0000 UTC m=+0.126145526 container died 69655ac4483b0a3f4a8ab506a852ff288c0da6671921140d5449303cb96d7356 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:46.007 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:45 smithi138 ceph-mon[25673]: Upgrade: Updating node-exporter.a (1/2) 2024-05-20T22:53:46.008 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:45 smithi138 ceph-mon[25673]: Deploying daemon node-exporter.a on smithi055 2024-05-20T22:53:46.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:45 smithi055 ceph-mon[24757]: Upgrade: Updating node-exporter.a (1/2) 2024-05-20T22:53:46.113 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:45 smithi055 ceph-mon[24757]: Deploying daemon node-exporter.a on smithi055 2024-05-20T22:53:46.113 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:45 smithi055 ceph-mon[29498]: Upgrade: Updating node-exporter.a (1/2) 2024-05-20T22:53:46.113 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:45 smithi055 ceph-mon[29498]: Deploying daemon node-exporter.a on smithi055 2024-05-20T22:53:46.113 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:45 smithi055 podman[76716]: 2024-05-20 22:53:45.844893895 +0000 UTC m=+0.473943117 container cleanup 69655ac4483b0a3f4a8ab506a852ff288c0da6671921140d5449303cb96d7356 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:46.113 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:45 smithi055 bash[76716]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a 2024-05-20T22:53:46.474 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:46 smithi055 podman[76728]: 2024-05-20 22:53:46.121197195 +0000 UTC m=+0.622732715 container remove 69655ac4483b0a3f4a8ab506a852ff288c0da6671921140d5449303cb96d7356 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:46.474 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:46 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-20T22:53:46.474 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:46 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-20T22:53:46.475 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:46 smithi055 systemd[1]: Stopped Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:53:46.475 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:46 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service: Consumed 5.513s CPU time. 2024-05-20T22:53:46.733 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:46 smithi055 systemd[1]: Starting Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:53:46.733 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:46 smithi055 bash[76855]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-05-20T22:53:47.023 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:46 smithi138 ceph-mon[25673]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:47.038 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:46 smithi055 ceph-mon[24757]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:47.039 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:46 smithi055 ceph-mon[29498]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:53:47.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:53:47] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-05-20T22:53:47.483 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:47 smithi055 bash[76855]: Getting image source signatures 2024-05-20T22:53:47.483 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:47 smithi055 bash[76855]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-05-20T22:53:47.483 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:47 smithi055 bash[76855]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-05-20T22:53:47.484 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:47 smithi055 bash[76855]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-05-20T22:53:48.697 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 bash[76855]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-05-20T22:53:48.697 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 bash[76855]: Writing manifest to image destination 2024-05-20T22:53:48.960 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-mon[24757]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 1 op/s 2024-05-20T22:53:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:48 smithi055 ceph-mon[29498]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 1 op/s 2024-05-20T22:53:48.961 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 podman[76855]: 2024-05-20 22:53:48.66870529 +0000 UTC m=+2.084897239 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-05-20T22:53:48.961 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 podman[76855]: 2024-05-20 22:53:48.741784838 +0000 UTC m=+2.157976834 container create b04e9536a7e1687577bb2b38a74a6299f2dcddd8a5fa64e163d2d0b9632680fd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:49.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:48 smithi138 ceph-mon[25673]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 1 op/s 2024-05-20T22:53:49.234 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 podman[76855]: 2024-05-20 22:53:48.969114448 +0000 UTC m=+2.385306374 container init b04e9536a7e1687577bb2b38a74a6299f2dcddd8a5fa64e163d2d0b9632680fd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:49.234 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 podman[76855]: 2024-05-20 22:53:48.97278691 +0000 UTC m=+2.388978853 container start b04e9536a7e1687577bb2b38a74a6299f2dcddd8a5fa64e163d2d0b9632680fd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T22:53:49.234 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.977Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-05-20T22:53:49.234 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.977Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-05-20T22:53:49.234 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.978Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-05-20T22:53:49.234 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.978Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.978Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.978Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=arp 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=bcache 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=bonding 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=btrfs 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=conntrack 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=cpu 2024-05-20T22:53:49.235 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=diskstats 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=dmi 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=edac 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=entropy 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=filefd 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=filesystem 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=hwmon 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=infiniband 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=ipvs 2024-05-20T22:53:49.236 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=loadavg 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=mdadm 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=meminfo 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=netclass 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=netdev 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=netstat 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=nfs 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=nfsd 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=nvme 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=os 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=pressure 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=rapl 2024-05-20T22:53:49.237 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=schedstat 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=selinux 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=sockstat 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=softnet 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=stat 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=tapestats 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=textfile 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=time 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=uname 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=vmstat 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=xfs 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=node_exporter.go:117 level=info collector=zfs 2024-05-20T22:53:49.238 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-05-20T22:53:49.239 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a[76906]: ts=2024-05-20T22:53:48.979Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-05-20T22:53:49.239 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:49 smithi055 bash[76855]: b04e9536a7e1687577bb2b38a74a6299f2dcddd8a5fa64e163d2d0b9632680fd 2024-05-20T22:53:49.239 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 22:53:49 smithi055 systemd[1]: Started Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:53:50.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:50 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:50.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:50 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:50.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:50 smithi138 ceph-mon[25673]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:53:50.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:50 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:50.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:50 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:50.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:50 smithi055 ceph-mon[29498]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:53:50.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:50 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:50.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:50 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:50.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:50 smithi055 ceph-mon[24757]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:53:51.406 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:51 smithi138 ceph-mon[25673]: Upgrade: Updating node-exporter.b (2/2) 2024-05-20T22:53:51.407 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:51 smithi138 ceph-mon[25673]: Deploying daemon node-exporter.b on smithi138 2024-05-20T22:53:51.407 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:51 smithi138 systemd[1]: Stopping Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:53:51.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:51 smithi055 ceph-mon[24757]: Upgrade: Updating node-exporter.b (2/2) 2024-05-20T22:53:51.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:51 smithi055 ceph-mon[24757]: Deploying daemon node-exporter.b on smithi138 2024-05-20T22:53:51.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:51 smithi055 ceph-mon[29498]: Upgrade: Updating node-exporter.b (2/2) 2024-05-20T22:53:51.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:51 smithi055 ceph-mon[29498]: Deploying daemon node-exporter.b on smithi138 2024-05-20T22:53:51.939 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:51 smithi138 podman[62179]: 2024-05-20 22:53:51.666599855 +0000 UTC m=+0.106702665 container died c17852f7fcd918089a90bccd986176be3321cb851273e7cd227db315066c9f1e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:53:52.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:52 smithi138 ceph-mon[25673]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:53:52.323 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:52 smithi138 podman[62179]: 2024-05-20 22:53:52.039033301 +0000 UTC m=+0.479136112 container cleanup c17852f7fcd918089a90bccd986176be3321cb851273e7cd227db315066c9f1e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:53:52.324 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:52 smithi138 bash[62179]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b 2024-05-20T22:53:52.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:52 smithi055 ceph-mon[24757]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:53:52.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:52 smithi055 ceph-mon[29498]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:53:52.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:52 smithi138 podman[62191]: 2024-05-20 22:53:52.332182757 +0000 UTC m=+0.665612465 container remove c17852f7fcd918089a90bccd986176be3321cb851273e7cd227db315066c9f1e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:53:52.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:52 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-20T22:53:52.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:52 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-20T22:53:52.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:52 smithi138 systemd[1]: Stopped Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:53:52.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:52 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service: Consumed 5.435s CPU time. 2024-05-20T22:53:53.106 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:52 smithi138 systemd[1]: Starting Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:53:53.106 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:52 smithi138 bash[62323]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-05-20T22:53:53.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:53 smithi138 bash[62323]: Getting image source signatures 2024-05-20T22:53:53.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:53 smithi138 bash[62323]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-05-20T22:53:53.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:53 smithi138 bash[62323]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-05-20T22:53:53.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:53 smithi138 bash[62323]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-05-20T22:53:54.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:54 smithi055 ceph-mon[24757]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-20T22:53:54.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:54.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:54 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1875878865' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:53:54.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:54 smithi055 ceph-mon[29498]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-20T22:53:54.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:54.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:54 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1875878865' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:53:54.782 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:54 smithi138 bash[62323]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-05-20T22:53:54.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:54 smithi138 ceph-mon[25673]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-20T22:53:54.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:53:54.782 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:54 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1875878865' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:53:55.078 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:54 smithi138 bash[62323]: Writing manifest to image destination 2024-05-20T22:53:55.330 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 podman[62323]: 2024-05-20 22:53:55.063006176 +0000 UTC m=+2.297576568 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-05-20T22:53:55.330 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 podman[62323]: 2024-05-20 22:53:55.128245525 +0000 UTC m=+2.362815894 container create f2df46ba361b007dfc7e0892983f293103c63323e400744de30c198e4c1491e1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:53:55.330 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 podman[62323]: 2024-05-20 22:53:55.338666464 +0000 UTC m=+2.573236836 container init f2df46ba361b007dfc7e0892983f293103c63323e400744de30c198e4c1491e1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:53:55.606 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 podman[62323]: 2024-05-20 22:53:55.342041205 +0000 UTC m=+2.576611577 container start f2df46ba361b007dfc7e0892983f293103c63323e400744de30c198e4c1491e1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.345Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.345Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.346Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.346Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=arp 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=bcache 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=bonding 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=btrfs 2024-05-20T22:53:55.607 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=conntrack 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=cpu 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=diskstats 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=dmi 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=edac 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=entropy 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=filefd 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=filesystem 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=hwmon 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=infiniband 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=ipvs 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=loadavg 2024-05-20T22:53:55.608 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=mdadm 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=meminfo 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=netclass 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=netdev 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=netstat 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=nfs 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=nfsd 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=nvme 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=os 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=pressure 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=rapl 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=schedstat 2024-05-20T22:53:55.609 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=selinux 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=sockstat 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=softnet 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=stat 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=tapestats 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=textfile 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=time 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=uname 2024-05-20T22:53:55.610 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=vmstat 2024-05-20T22:53:55.611 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=xfs 2024-05-20T22:53:55.611 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=node_exporter.go:117 level=info collector=zfs 2024-05-20T22:53:55.611 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-05-20T22:53:55.611 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b[62372]: ts=2024-05-20T22:53:55.347Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-05-20T22:53:55.611 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 bash[62323]: f2df46ba361b007dfc7e0892983f293103c63323e400744de30c198e4c1491e1 2024-05-20T22:53:55.611 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 22:53:55 smithi138 systemd[1]: Started Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:53:56.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:56 smithi055 ceph-mon[24757]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-20T22:53:56.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:56 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:56.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:56 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:56.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:56 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:56.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:56 smithi055 ceph-mon[29498]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-20T22:53:56.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:56 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:56.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:56 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:56.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:56 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:56.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:56 smithi138 ceph-mon[25673]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-20T22:53:56.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:56 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:56.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:56 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:53:56.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:56 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:53:57.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:53:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:53:57] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-05-20T22:53:58.763 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:53:58 smithi138 ceph-mon[25673]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:53:58.842 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:53:58 smithi055 ceph-mon[24757]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:53:58.842 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:53:58 smithi055 ceph-mon[29498]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:54:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:00 smithi138 ceph-mon[25673]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:00 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:00 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:00 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:00 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[24757]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:01.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[29498]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:01.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:01.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:00 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:01 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:01 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:01 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:01 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:01 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:01 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.036 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:01 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.036 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:01 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:01 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:01 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:01 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:01 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:02.960 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:02 smithi055 ceph-mon[24757]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:02.960 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:02 smithi055 ceph-mon[29498]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:03.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:02 smithi138 ceph-mon[25673]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:05.204 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:05.204 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.204 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.204 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.204 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.204 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:05.205 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:54:05.206 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.206 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:54:05.206 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:04 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:54:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:54:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:04 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:06.163 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:05 smithi055 ceph-mon[24757]: from='client.35225 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:54:06.163 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:05 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:54:06.163 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:05 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:54:06.163 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:05 smithi055 ceph-mon[29498]: from='client.35225 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:54:06.163 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:05 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:54:06.164 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:05 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:54:06.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:05 smithi138 ceph-mon[25673]: from='client.35225 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:54:06.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:05 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:54:06.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:05 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:54:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:06 smithi055 ceph-mon[24757]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:06.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:06 smithi055 ceph-mon[29498]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:07.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:06 smithi138 ceph-mon[25673]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:07.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:54:07] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-05-20T22:54:08.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:54:07.838+0000 7f35b53ff700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:08.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:54:08.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:08.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:08.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:08.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return future.result(timeout) 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.__get_result() 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: raise self._exception 2024-05-20T22:54:08.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: use_current_daemon_image=reconfig, 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:08.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:08.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.746 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:08.746 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:08.746 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:08.747 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:08.748 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: return future.result(timeout) 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.__get_result() 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: raise self._exception 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:08.749 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:08.750 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.751 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.752 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.752 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.752 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.752 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.752 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:08 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:08.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:08.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:08.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: return future.result(timeout) 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.__get_result() 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: raise self._exception 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:08.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:08.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:08.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:08.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: return future.result(timeout) 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.__get_result() 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: raise self._exception 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:08.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:08.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:08.993 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:08.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:08 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:09.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:09 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:09.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:09 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:09.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:09 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:10.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:10 smithi138 ceph-mon[25673]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:10.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:10 smithi138 ceph-mon[25673]: Upgrade: Updating prometheus.a 2024-05-20T22:54:10.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:10 smithi138 ceph-mon[25673]: Deploying daemon prometheus.a on smithi138 2024-05-20T22:54:10.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:10 smithi055 ceph-mon[24757]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:10.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:10 smithi055 ceph-mon[24757]: Upgrade: Updating prometheus.a 2024-05-20T22:54:10.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:10 smithi055 ceph-mon[24757]: Deploying daemon prometheus.a on smithi138 2024-05-20T22:54:10.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:10 smithi055 ceph-mon[29498]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:10.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:10 smithi055 ceph-mon[29498]: Upgrade: Updating prometheus.a 2024-05-20T22:54:10.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:10 smithi055 ceph-mon[29498]: Deploying daemon prometheus.a on smithi138 2024-05-20T22:54:12.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:12 smithi138 ceph-mon[25673]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:12.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:12 smithi055 ceph-mon[24757]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:12.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:12 smithi055 ceph-mon[29498]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:13.742 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (13m) 14s ago 21m 22.5M - 0.23.0 ba2b418f427c a0f540c0ca04 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (13m) 14s ago 21m 50.9M - 8.3.5 dad864ee21e9 179b97bd7400 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (20s) 14s ago 20m 45.9M - 3.5 e1d6a67b021e f7f69dd640f2 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (44s) 14s ago 26m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (10m) 14s ago 27m 503M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (27m) 14s ago 28m 74.8M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (26m) 14s ago 26m 57.7M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (26m) 14s ago 26m 57.4M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (25s) 14s ago 22m 3590k - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (19s) 14s ago 22m 3569k - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:54:14.419 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (25m) 14s ago 25m 55.7M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (25m) 14s ago 25m 55.3M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (25m) 14s ago 25m 53.4M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (24m) 14s ago 24m 53.5M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (24m) 14s ago 24m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (23m) 14s ago 23m 52.4M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (23m) 14s ago 23m 52.6M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (23m) 14s ago 23m 53.1M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (49s) 14s ago 22m 64.3M - 2.33.4 514e6a882f6e b3ca603550e1 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (20m) 14s ago 20m 75.0M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:54:14.420 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (20m) 14s ago 20m 78.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:54:14.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:14 smithi055 ceph-mon[24757]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:14.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:14 smithi055 ceph-mon[24757]: from='client.25480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:14.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:14 smithi055 ceph-mon[29498]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:14.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:14 smithi055 ceph-mon[29498]: from='client.25480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:14.832 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:54:14.832 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:54:14.832 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:54:14.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:54:14.832 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:54:14.832 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:54:14.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:54:14.832 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:54:14.833 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:54:14.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:14 smithi138 ceph-mon[25673]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:14.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:14 smithi138 ceph-mon[25673]: from='client.25480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:15.184 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:54:15.184 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:54:15.184 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:54:15.184 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-20T22:54:15.185 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [ 2024-05-20T22:54:15.185 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr" 2024-05-20T22:54:15.185 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:54:15.185 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "2/2 daemons upgraded", 2024-05-20T22:54:15.185 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading prometheus daemons", 2024-05-20T22:54:15.185 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:54:15.185 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:54:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:15 smithi138 ceph-mon[25673]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:15 smithi138 ceph-mon[25673]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:15 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/1180377370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:15.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:15 smithi055 ceph-mon[24757]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:15.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:15 smithi055 ceph-mon[24757]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:15.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:15 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/1180377370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:15.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:15 smithi055 ceph-mon[29498]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:15.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:15 smithi055 ceph-mon[29498]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:15.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:15 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/1180377370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:16 smithi138 ceph-mon[25673]: from='client.25504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:16 smithi138 ceph-mon[25673]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:16 smithi055 ceph-mon[24757]: from='client.25504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:16 smithi055 ceph-mon[24757]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:16.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:16 smithi055 ceph-mon[29498]: from='client.25504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:16.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:16 smithi055 ceph-mon[29498]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:17.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:54:17] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-05-20T22:54:18.850 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:18 smithi138 ceph-mon[25673]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:18.924 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:18 smithi055 ceph-mon[24757]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:18.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:18 smithi055 ceph-mon[29498]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:19.810 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 systemd[1]: Stopping Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:54:20.106 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.818Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-20T22:54:20.106 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.818Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-20T22:54:20.106 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.818Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-20T22:54:20.106 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.818Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-20T22:54:20.106 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.818Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-20T22:54:20.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.818Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-20T22:54:20.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.818Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-20T22:54:20.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.818Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-20T22:54:20.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.818Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-20T22:54:20.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.819Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-20T22:54:20.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.819Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-20T22:54:20.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[60315]: ts=2024-05-20T22:54:19.819Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-20T22:54:20.107 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:19 smithi138 podman[63852]: 2024-05-20 22:54:19.828593038 +0000 UTC m=+0.116996829 container died b3ca603550e1f2ae479b39f83a21923148d8b04e8914a4dee5f74031a290f199 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:20.444 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:20 smithi138 podman[63852]: 2024-05-20 22:54:20.218334341 +0000 UTC m=+0.506738122 container cleanup b3ca603550e1f2ae479b39f83a21923148d8b04e8914a4dee5f74031a290f199 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:20.444 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:20 smithi138 bash[63852]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a 2024-05-20T22:54:20.772 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:20 smithi138 ceph-mon[25673]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:20.772 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:20 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2170420117' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:54:20.773 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:20 smithi138 podman[63865]: 2024-05-20 22:54:20.452877825 +0000 UTC m=+0.626714077 container remove b3ca603550e1f2ae479b39f83a21923148d8b04e8914a4dee5f74031a290f199 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:20.773 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:20 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-20T22:54:20.773 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:20 smithi138 systemd[1]: Stopped Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:54:20.773 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:20 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service: Consumed 1.319s CPU time. 2024-05-20T22:54:20.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:20 smithi055 ceph-mon[24757]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:20.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:20 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2170420117' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:54:20.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:20 smithi055 ceph-mon[29498]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:20.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:20 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2170420117' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:54:21.084 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:20 smithi138 systemd[1]: Starting Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:54:21.085 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:20 smithi138 podman[63991]: 2024-05-20 22:54:20.897733508 +0000 UTC m=+0.018047640 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-05-20T22:54:21.085 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:20 smithi138 podman[63991]: 2024-05-20 22:54:20.995466717 +0000 UTC m=+0.115780837 container create c7fec3619ff9ac2352dd7bf8ed4514bdafd4c6230096af8e58452b87212e3646 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:21.606 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 podman[63991]: 2024-05-20 22:54:21.214554839 +0000 UTC m=+0.334868963 container init c7fec3619ff9ac2352dd7bf8ed4514bdafd4c6230096af8e58452b87212e3646 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:21.606 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 podman[63991]: 2024-05-20 22:54:21.217321272 +0000 UTC m=+0.337635391 container start c7fec3619ff9ac2352dd7bf8ed4514bdafd4c6230096af8e58452b87212e3646 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:21.606 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.241Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.241Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.241Z caller=main.go:570 level=info host_details="(Linux 5.14.0-447.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 7 10:46:15 UTC 2024 x86_64 smithi138 (none))" 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.241Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.241Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.243Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.243Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.245Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.245Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.246Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.246Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.093µs 2024-05-20T22:54:21.607 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.246Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.254Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.270Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.278Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.281Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.287Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.288Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.288Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.288Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.073µs wal_replay_duration=41.643096ms wbl_replay_duration=512ns total_replay_duration=41.692905ms 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 bash[63991]: c7fec3619ff9ac2352dd7bf8ed4514bdafd4c6230096af8e58452b87212e3646 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 systemd[1]: Started Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.292Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.292Z caller=main.go:1029 level=info msg="TSDB started" 2024-05-20T22:54:21.608 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.292Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-20T22:54:21.609 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.312Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.770434ms db_storage=1.211µs remote_storage=1.329µs web_handler=523ns query_engine=848ns scrape=570.978µs scrape_sd=134.982µs notify=16.391µs notify_sd=16.544µs rules=18.487157ms tracing=5.66µs 2024-05-20T22:54:21.609 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.312Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-05-20T22:54:21.609 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 22:54:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T22:54:21.312Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-05-20T22:54:22.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:22 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:22.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:22 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:22.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:22 smithi138 ceph-mon[25673]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:54:22.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:22 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:22.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:22 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:22.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:22 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:22.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:22 smithi055 ceph-mon[24757]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:54:22.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:22 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:22.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:22 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:22.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:22 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:22.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:22 smithi055 ceph-mon[29498]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:54:22.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:22 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:24.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:24 smithi138 ceph-mon[25673]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:24.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:24.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:24 smithi055 ceph-mon[24757]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:24.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:24.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:24 smithi055 ceph-mon[29498]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:24.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:26.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:26 smithi055 ceph-mon[29498]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:26.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:26 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:26.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:26 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:26.785 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:26 smithi055 ceph-mon[24757]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:26.786 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:26 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:26.786 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:26 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:26.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:26 smithi138 ceph-mon[25673]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:26.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:26 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:26.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:26 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:27.560 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:54:27] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-05-20T22:54:27.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:27.799 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:27.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:27.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:27.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:27.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:28.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:28 smithi138 ceph-mon[25673]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:28.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:28 smithi055 ceph-mon[24757]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:28.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:28 smithi055 ceph-mon[29498]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:31.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: from='client.25522 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:54:31.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:31.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:31.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:31.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:31.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:54:31.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:31.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:54:31.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:54:31.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:31.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:30 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:54:31.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: from='client.25522 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:54:31.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:31.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:31.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: from='client.25522 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:31.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:31.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:31.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:54:31.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:31.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:54:31.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:54:31.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:31.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:30 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:54:33.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:32 smithi138 ceph-mon[25673]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:33.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:32 smithi055 ceph-mon[24757]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:33.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:32 smithi055 ceph-mon[29498]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:33.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:54:33.024+0000 7f35b53ff700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:54:33.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:33.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return future.result(timeout) 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.__get_result() 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: raise self._exception 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: use_current_daemon_image=reconfig, 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:54:33.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:33.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:33.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:33.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:33.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:33.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:54:33.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:33.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:54:33.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:33.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:33.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:33 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:54:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:33 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:34.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:34 smithi055 ceph-mon[24757]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:34.983 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:34.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:34.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: return future.result(timeout) 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.__get_result() 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: raise self._exception 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:34.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:34.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:34 smithi055 ceph-mon[24757]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:34 smithi055 ceph-mon[24757]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:34 smithi055 ceph-mon[29498]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:34.988 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:34.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: return future.result(timeout) 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:34.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.__get_result() 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: raise self._exception 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:34.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:34.992 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:34 smithi055 ceph-mon[29498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:54:34.993 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:34 smithi055 ceph-mon[29498]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:34 smithi138 ceph-mon[25673]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: return future.result(timeout) 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.__get_result() 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:35.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: raise self._exception 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:54:35.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:35.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:34 smithi138 ceph-mon[25673]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:54:35.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:34 smithi138 ceph-mon[25673]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:36.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:35 smithi138 ceph-mon[25673]: Upgrade: Updating alertmanager.a 2024-05-20T22:54:36.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:35 smithi138 ceph-mon[25673]: Deploying daemon alertmanager.a on smithi055 2024-05-20T22:54:36.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:35 smithi055 ceph-mon[24757]: Upgrade: Updating alertmanager.a 2024-05-20T22:54:36.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:35 smithi055 ceph-mon[24757]: Deploying daemon alertmanager.a on smithi055 2024-05-20T22:54:36.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:35 smithi055 ceph-mon[29498]: Upgrade: Updating alertmanager.a 2024-05-20T22:54:36.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:35 smithi055 ceph-mon[29498]: Deploying daemon alertmanager.a on smithi055 2024-05-20T22:54:37.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:36 smithi055 ceph-mon[24757]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:37.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:36 smithi055 ceph-mon[29498]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:37.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:36 smithi138 ceph-mon[25673]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:37.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:54:37] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-05-20T22:54:39.227 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:38 smithi055 ceph-mon[24757]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:39.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:38 smithi055 ceph-mon[29498]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:39.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:38 smithi138 ceph-mon[25673]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:54:40.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:39 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:40.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:39 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:40.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:39 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:40.945 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:40 smithi055 ceph-mon[24757]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:40.945 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:40 smithi055 ceph-mon[29498]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:41.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:40 smithi138 ceph-mon[25673]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:42.483 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:42 smithi055 systemd[1]: Stopping Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:54:42.806 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[58850]: level=info ts=2024-05-20T22:54:42.496Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-20T22:54:42.806 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:42 smithi055 podman[79312]: 2024-05-20 22:54:42.537852419 +0000 UTC m=+0.157129957 container died a0f540c0ca049b3c4627ff362461ce6088bcde10950e8a3b04a52d15ee6e5878 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:43.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:42 smithi055 ceph-mon[24757]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:43.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:42 smithi055 ceph-mon[29498]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:43.156 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:42 smithi055 podman[79312]: 2024-05-20 22:54:42.913402718 +0000 UTC m=+0.532680248 container cleanup a0f540c0ca049b3c4627ff362461ce6088bcde10950e8a3b04a52d15ee6e5878 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:43.157 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:42 smithi055 bash[79312]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a 2024-05-20T22:54:43.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:42 smithi138 ceph-mon[25673]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:54:43.451 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:43 smithi055 podman[79325]: 2024-05-20 22:54:43.164881863 +0000 UTC m=+0.621024361 container remove a0f540c0ca049b3c4627ff362461ce6088bcde10950e8a3b04a52d15ee6e5878 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:43.451 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:43 smithi055 podman[79325]: 2024-05-20 22:54:43.248803408 +0000 UTC m=+0.704945858 volume remove 74a641217cebd0b4590dd0a069dbedc1d57bad709e581e758f69f0a1cff11003 2024-05-20T22:54:43.987 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:43 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-20T22:54:43.987 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:43 smithi055 systemd[1]: Stopped Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:54:43.987 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:43 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@alertmanager.a.service: Consumed 1.610s CPU time. 2024-05-20T22:54:44.268 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 systemd[1]: Starting Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:54:44.549 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 podman[79593]: 2024-05-20 22:54:44.350629451 +0000 UTC m=+0.022189225 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-05-20T22:54:44.549 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 podman[79593]: 2024-05-20 22:54:44.457177728 +0000 UTC m=+0.128737502 volume create 3bc418356b6b438fa59fc9c59799e75990924a293e951670771e62ab6154cb0f 2024-05-20T22:54:44.802 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 podman[79593]: 2024-05-20 22:54:44.5578014 +0000 UTC m=+0.229361178 container create df8a6bb3b4f5bc5c892efc00497a4fd42385d42d5051ee43ded926a97e61ff4d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:45.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-mon[24757]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:45.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2801417173' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:54:45.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:44 smithi055 ceph-mon[29498]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:45.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:44 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2801417173' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 podman[79593]: 2024-05-20 22:54:44.810656356 +0000 UTC m=+0.482216121 container init df8a6bb3b4f5bc5c892efc00497a4fd42385d42d5051ee43ded926a97e61ff4d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 podman[79593]: 2024-05-20 22:54:44.813748531 +0000 UTC m=+0.485308305 container start df8a6bb3b4f5bc5c892efc00497a4fd42385d42d5051ee43ded926a97e61ff4d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:44.825Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:44.825Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:44.827Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.55 port=9094 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:44.828Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:44.851Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:44.852Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:44.855Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-05-20T22:54:45.234 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:44.855Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-05-20T22:54:45.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 bash[79593]: df8a6bb3b4f5bc5c892efc00497a4fd42385d42d5051ee43ded926a97e61ff4d 2024-05-20T22:54:45.235 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:44 smithi055 systemd[1]: Started Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:54:45.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:44 smithi138 ceph-mon[25673]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:54:45.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:44 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2801417173' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:54:45.575 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:54:46.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:45 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:46.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:45 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:46.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:45 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:46.166 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:45 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:46.166 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:45 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:46.167 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:45 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:46.167 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:45 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:46.167 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:45 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:46.167 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:45 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:46.265 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:54:46.266 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 starting - - - - 2024-05-20T22:54:46.266 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (14m) 20s ago 21m 49.5M - 8.3.5 dad864ee21e9 179b97bd7400 2024-05-20T22:54:46.266 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (52s) 46s ago 21m 45.9M - 3.5 e1d6a67b021e f7f69dd640f2 2024-05-20T22:54:46.266 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (76s) 20s ago 26m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:54:46.266 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (11m) 46s ago 28m 503M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:54:46.267 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (28m) 46s ago 28m 74.8M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:54:46.267 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (27m) 20s ago 27m 62.5M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:54:46.267 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (27m) 46s ago 27m 57.4M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:54:46.267 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (57s) 46s ago 22m 3590k - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:54:46.267 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (51s) 20s ago 22m 8456k - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:54:46.267 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (26m) 46s ago 26m 55.7M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:54:46.267 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (26m) 46s ago 26m 55.3M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:54:46.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (25m) 46s ago 25m 53.4M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:54:46.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (25m) 46s ago 25m 53.5M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:54:46.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (24m) 20s ago 24m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:54:46.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (24m) 20s ago 24m 52.4M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:54:46.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (24m) 20s ago 24m 52.6M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:54:46.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (23m) 20s ago 23m 53.1M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:54:46.268 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (25s) 20s ago 23m 46.1M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:54:46.269 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (21m) 46s ago 21m 75.0M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:54:46.269 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (21m) 20s ago 21m 78.9M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:54:46.690 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:54:46.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:54:46.692 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:54:46.692 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:54:46.692 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:54:46.692 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:54:46.692 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:54:46.962 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:46 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:46.829Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000154792s 2024-05-20T22:54:47.043 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [ 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr" 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "2/2 daemons upgraded", 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading alertmanager daemons", 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:54:47.044 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:54:47.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:46 smithi055 ceph-mon[24757]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:54:47.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:46 smithi055 ceph-mon[24757]: from='client.15666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:47.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:46 smithi055 ceph-mon[24757]: from='client.25561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:47.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:46 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/676612804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:47.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:46 smithi055 ceph-mon[29498]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:54:47.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:46 smithi055 ceph-mon[29498]: from='client.15666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:47.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:46 smithi055 ceph-mon[29498]: from='client.25561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:47.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:46 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/676612804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:47.268 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:46 smithi138 ceph-mon[25673]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:54:47.268 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:46 smithi138 ceph-mon[25673]: from='client.15666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:47.268 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:46 smithi138 ceph-mon[25673]: from='client.25561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:47.268 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:46 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/676612804' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:47.574 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:54:47] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-05-20T22:54:48.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:47 smithi055 ceph-mon[24757]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:47 smithi055 ceph-mon[29498]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:48.294 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:47 smithi138 ceph-mon[25673]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:49.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:48 smithi055 ceph-mon[24757]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:49.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:48 smithi055 ceph-mon[24757]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:54:49.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:48 smithi055 ceph-mon[29498]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:49.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:48 smithi055 ceph-mon[29498]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:54:49.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:48 smithi138 ceph-mon[25673]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:54:49.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:48 smithi138 ceph-mon[25673]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T22:54:50.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[29498]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-20T22:54:50.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[24757]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-20T22:54:50.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:50 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:50 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:50 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:50 smithi138 ceph-mon[25673]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-05-20T22:54:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:50 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:50.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:50 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:51.804 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:51 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:51.804 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:51 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:51.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:51 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:51.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:51 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:52.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:51 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:52.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:51 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:52.974 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:52 smithi055 ceph-mon[24757]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:54:52.974 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:52 smithi055 ceph-mon[29498]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:54:53.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:52 smithi138 ceph-mon[25673]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:54:54.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[24757]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:54:54.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[29498]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:54.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:54.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:54.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:54.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:54:54.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:54.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:54:54.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:54.985 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 22:54:54 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T22:54:54.832Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003509312s 2024-05-20T22:54:55.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:54 smithi138 ceph-mon[25673]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-20T22:54:55.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:54:55.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:55.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:55.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:55.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:54:55.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:54:55.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:54:55.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:54:55.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:55 smithi055 ceph-mon[24757]: from='client.15654 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:54:55.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:55 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:54:55.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:55 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:54:55.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:55 smithi055 ceph-mon[29498]: from='client.15654 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:54:55.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:55 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:54:55.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:55 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:54:56.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:55 smithi138 ceph-mon[25673]: from='client.15654 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:54:56.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:55 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:54:56.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:55 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:54:56.785 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:56 smithi055 ceph-mon[24757]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:54:56.786 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:56 smithi055 ceph-mon[29498]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:54:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:56 smithi138 ceph-mon[25673]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-05-20T22:54:57.565 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:54:57] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-05-20T22:54:58.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:54:57.889+0000 7f35b53ff700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:54:58.234 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:58.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:58.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:58.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:58.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:58.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:58.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:58.235 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return future.result(timeout) 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:58.236 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.__get_result() 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: raise self._exception 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: use_current_daemon_image=reconfig, 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.237 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:58.238 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:58.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.239 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:54:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:58.984 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.985 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: return future.result(timeout) 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.__get_result() 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:58.986 INFO:journalctl@ceph.mon.a.smithi055.stdout: raise self._exception 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.987 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:58.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:58.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:58.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:58.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:58.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:58.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:58.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:58.988 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:58.989 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:58.990 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: return future.result(timeout) 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.__get_result() 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:58.991 INFO:journalctl@ceph.mon.c.smithi055.stdout: raise self._exception 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:54:58.992 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:58.993 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:54:58.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:58.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:54:58 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:59.107 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:54:59.108 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: return future.result(timeout) 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.__get_result() 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: raise self._exception 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:54:59.109 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:54:59.110 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:54:59.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:54:59.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:54:58 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:00.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:00 smithi055 ceph-mon[29498]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:00.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:00 smithi055 ceph-mon[29498]: Upgrade: Updating grafana.a 2024-05-20T22:55:00.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:00 smithi055 ceph-mon[29498]: Deploying daemon grafana.a on smithi138 2024-05-20T22:55:00.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:00 smithi055 ceph-mon[24757]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:00.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:00 smithi055 ceph-mon[24757]: Upgrade: Updating grafana.a 2024-05-20T22:55:00.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:00 smithi055 ceph-mon[24757]: Deploying daemon grafana.a on smithi138 2024-05-20T22:55:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:00 smithi138 ceph-mon[25673]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:00 smithi138 ceph-mon[25673]: Upgrade: Updating grafana.a 2024-05-20T22:55:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:00 smithi138 ceph-mon[25673]: Deploying daemon grafana.a on smithi138 2024-05-20T22:55:02.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:02 smithi055 ceph-mon[24757]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:02.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:02 smithi055 ceph-mon[29498]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:03.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:02 smithi138 ceph-mon[25673]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:04 smithi055 ceph-mon[24757]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:04.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:04 smithi055 ceph-mon[29498]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:04 smithi138 ceph-mon[25673]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:07.068 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:06 smithi138 ceph-mon[25673]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:07.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:06 smithi055 ceph-mon[24757]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:07.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:06 smithi055 ceph-mon[29498]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:07.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:55:07] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-05-20T22:55:09.074 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:08 smithi055 ceph-mon[24757]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:55:09.074 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:08 smithi055 ceph-mon[29498]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:55:09.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:08 smithi138 ceph-mon[25673]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:55:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:09 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:09 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3595994659' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:55:09.856 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:09 smithi138 systemd[1]: Stopping Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:55:09.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:09 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:09.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:09 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3595994659' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:55:09.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:09 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:09.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:09 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3595994659' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:55:10.123 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:09 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[49301]: t=2024-05-20T22:55:09+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-05-20T22:55:10.123 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:09 smithi138 podman[66058]: 2024-05-20 22:55:09.87538103 +0000 UTC m=+0.136500122 container died 179b97bd74004e079a70c30c88db3819f289c15c4147555f756ab768dec8f7ca (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, version=8.5, maintainer=Paul Cuzner , architecture=x86_64, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, name=ubi8, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, distribution-scope=public) 2024-05-20T22:55:10.482 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:10 smithi138 podman[66058]: 2024-05-20 22:55:10.231191642 +0000 UTC m=+0.492310727 container cleanup 179b97bd74004e079a70c30c88db3819f289c15c4147555f756ab768dec8f7ca (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.tags=base rhel8, distribution-scope=public, release=236.1648460182, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., description=Ceph Grafana Container, version=8.5, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-05-20T22:55:10.482 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:10 smithi138 bash[66058]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a 2024-05-20T22:55:10.765 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:10 smithi138 podman[66070]: 2024-05-20 22:55:10.490843097 +0000 UTC m=+0.611794647 container remove 179b97bd74004e079a70c30c88db3819f289c15c4147555f756ab768dec8f7ca (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.openshift.tags=base rhel8, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, release=236.1648460182, architecture=x86_64, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, distribution-scope=public, vcs-type=git) 2024-05-20T22:55:10.765 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:10 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-20T22:55:10.765 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:10 smithi138 systemd[1]: Stopped Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:55:10.766 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:10 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@grafana.a.service: Consumed 5.560s CPU time. 2024-05-20T22:55:11.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:10 smithi138 ceph-mon[25673]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:55:11.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:10 smithi138 systemd[1]: Starting Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:55:11.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:10 smithi055 ceph-mon[24757]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:55:11.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:10 smithi055 ceph-mon[29498]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:55:11.356 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 podman[66200]: 2024-05-20 22:55:10.968149223 +0000 UTC m=+0.017682879 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-05-20T22:55:11.356 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 podman[66200]: 2024-05-20 22:55:11.092772769 +0000 UTC m=+0.142306448 container create c12c9c6bb205769261cf19338518ec15b944fec1674a60fdfcf72de8b3ca1b09 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-20T22:55:11.609 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 podman[66200]: 2024-05-20 22:55:11.370154128 +0000 UTC m=+0.419687784 container init c12c9c6bb205769261cf19338518ec15b944fec1674a60fdfcf72de8b3ca1b09 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-20T22:55:11.609 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 podman[66200]: 2024-05-20 22:55:11.373348005 +0000 UTC m=+0.422881664 container start c12c9c6bb205769261cf19338518ec15b944fec1674a60fdfcf72de8b3ca1b09 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-20T22:55:11.609 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 bash[66200]: c12c9c6bb205769261cf19338518ec15b944fec1674a60fdfcf72de8b3ca1b09 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 systemd[1]: Started Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526542559Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526728586Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.52673993Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526743447Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.5267462Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526748947Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526751614Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526754166Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526757306Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526760161Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-20T22:55:11.610 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526762909Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526765484Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526768038Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526770825Z level=info msg="Path Home" path=/usr/share/grafana 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526773387Z level=info msg="Path Data" path=/var/lib/grafana 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526775973Z level=info msg="Path Logs" path=/var/log/grafana 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526778319Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526780814Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=settings t=2024-05-20T22:55:11.526783349Z level=info msg="App mode production" 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=sqlstore t=2024-05-20T22:55:11.526822823Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=sqlstore t=2024-05-20T22:55:11.526834132Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.53819484Z level=info msg="Starting DB migrations" 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.541687531Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.552889531Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-05-20T22:55:11.611 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.554041382Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-05-20T22:55:11.612 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.554377719Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-05-20T22:55:11.612 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.555588329Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-05-20T22:55:11.612 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.556728244Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-05-20T22:55:11.612 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.557802875Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-05-20T22:55:11.612 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.558105823Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-05-20T22:55:11.612 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.558903093Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-05-20T22:55:11.612 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.559425227Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-05-20T22:55:11.612 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.559757755Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.560857838Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.561170077Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.562236061Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.563286323Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.564401044Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.564696973Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.565788235Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.566860763Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.567149316Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-05-20T22:55:11.613 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.568180883Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.569264126Z level=info msg="Executing migration" id="create provenance_type table" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.569683878Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.572355578Z level=info msg="Executing migration" id="create alert_image table" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.574849402Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.577394108Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.577713976Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.578135595Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.578599938Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.578675047Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.578913178Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.579377059Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.57980742Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-05-20T22:55:11.614 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.580064213Z level=info msg="Executing migration" id="create secrets table" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.580493068Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.586711205Z level=info msg="Executing migration" id="add name column into data_keys" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.587871417Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.58819402Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.594622735Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.600954579Z level=info msg="Executing migration" id="add column hidden to role table" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.602116052Z level=info msg="Executing migration" id="create query_history table v1" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.602528404Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.602962065Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.603218987Z level=info msg="Executing migration" id="teams permissions migration" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.603649111Z level=info msg="Executing migration" id="dashboard permissions" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.604885227Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.605584365Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-05-20T22:55:11.615 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.605896498Z level=info msg="Executing migration" id="alerting notification permissions" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.606211962Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.606598432Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.607017854Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.608141201Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.608445618Z level=info msg="Executing migration" id="create correlation table v1" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.608879512Z level=info msg="Executing migration" id="add index correlations.uid" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.609300555Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.609724405Z level=info msg="Executing migration" id="add correlation config column" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.610849236Z level=info msg="Executing migration" id="create entity_events table" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.611223352Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.613662509Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.61374Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-05-20T22:55:11.616 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.614008166Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-20T22:55:11.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.614078061Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-20T22:55:11.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.614421217Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-05-20T22:55:11.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.614848672Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-05-20T22:55:11.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.617532094Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-05-20T22:55:11.617 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.618045049Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-20T22:55:12.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.620821279Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-05-20T22:55:12.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.623372075Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-05-20T22:55:12.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.625956391Z level=info msg="Executing migration" id="Drop public config table" 2024-05-20T22:55:12.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.628484825Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-05-20T22:55:12.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.631063511Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-05-20T22:55:12.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.633667031Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-05-20T22:55:12.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.636295512Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-05-20T22:55:12.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.639248547Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.643332025Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.64449375Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.645716797Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.646488601Z level=info msg="Executing migration" id="add share column" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.647755434Z level=info msg="Executing migration" id="create default alerting folders" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.648492527Z level=info msg="Executing migration" id="create file table" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.649312626Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.65009531Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.650914642Z level=info msg="Executing migration" id="create file_meta table" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.651825984Z level=info msg="Executing migration" id="file table idx: path key" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.652621913Z level=info msg="Executing migration" id="set path collation in file table" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.65349513Z level=info msg="Executing migration" id="managed permissions migration" 2024-05-20T22:55:12.111 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.654618728Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.655436392Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.65628794Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.657537396Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.658348894Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.659179555Z level=info msg="Executing migration" id="update group index for alert rules" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.660027421Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.660823105Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.661672307Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.662906367Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.664145392Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.664926491Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.678271888Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.679073335Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-05-20T22:55:12.112 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.681816648Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.686513291Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.689324237Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.692156379Z level=info msg="Executing migration" id="create folder table" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.694828388Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.697386084Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.700061443Z level=info msg="Executing migration" id="Update folder title length" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.702731668Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=migrator t=2024-05-20T22:55:11.705447185Z level=info msg="migrations completed" performed=105 skipped=376 duration=163.792765ms 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=sqlstore t=2024-05-20T22:55:11.705820453Z level=info msg="Created default organization" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=plugin.finder t=2024-05-20T22:55:11.73891565Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=secrets t=2024-05-20T22:55:11.739038891Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=query_data t=2024-05-20T22:55:11.74278437Z level=info msg="Query Service initialization" 2024-05-20T22:55:12.113 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=live.push_http t=2024-05-20T22:55:11.745513131Z level=info msg="Live Push Gateway initialization" 2024-05-20T22:55:12.674 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:12 smithi138 ceph-mon[25673]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:55:12.675 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:12 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:12.675 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:12 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:12.675 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:12 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:12.675 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=infra.usagestats.collector t=2024-05-20T22:55:12.403499284Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-05-20T22:55:12.675 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=provisioning.datasources t=2024-05-20T22:55:12.405340301Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-05-20T22:55:12.675 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=provisioning.datasources t=2024-05-20T22:55:12.405349426Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-05-20T22:55:12.675 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=provisioning.datasources t=2024-05-20T22:55:12.405446287Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-20T22:55:12.675 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=provisioning.alerting t=2024-05-20T22:55:12.428005576Z level=info msg="starting to provision alerting" 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=provisioning.alerting t=2024-05-20T22:55:12.428021233Z level=info msg="finished to provision alerting" 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=grafanaStorageLogger t=2024-05-20T22:55:12.428323749Z level=info msg="storage starting" 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=ngalert.state.manager t=2024-05-20T22:55:12.428350876Z level=info msg="Warming state cache for startup" 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=http.server t=2024-05-20T22:55:12.431267387Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=sqlstore.transactions t=2024-05-20T22:55:12.46531656Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=ngalert.state.manager t=2024-05-20T22:55:12.47009281Z level=info msg="State cache has been initialized" states=0 duration=41.741431ms 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=ticker t=2024-05-20T22:55:12.470138737Z level=info msg=starting first_tick=2024-05-20T22:55:20Z 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=ngalert.multiorg.alertmanager t=2024-05-20T22:55:12.470148695Z level=info msg="starting MultiOrg Alertmanager" 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=sqlstore.transactions t=2024-05-20T22:55:12.476541554Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-05-20T22:55:12.676 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 22:55:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=sqlstore.transactions t=2024-05-20T22:55:12.487965498Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-05-20T22:55:12.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:12 smithi055 ceph-mon[24757]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:55:12.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:12 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:12.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:12 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:12.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:12 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:12.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:12 smithi055 ceph-mon[29498]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:55:12.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:12.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:12.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:14.851 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:14 smithi138 ceph-mon[25673]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:55:14.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:14 smithi055 ceph-mon[29498]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:55:14.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:14 smithi055 ceph-mon[24757]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:55:17.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:16 smithi055 ceph-mon[24757]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:17.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:16 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:17.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:16 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:17.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:16 smithi055 ceph-mon[29498]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:17.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:16 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:17.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:16 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:17.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:16 smithi138 ceph-mon[25673]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:17.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:16 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:17.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:16 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:17.425 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:55:17.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:55:17] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-05-20T22:55:18.107 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:55:18.107 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (33s) 28s ago 22m 17.5M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T22:55:18.107 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (7s) 2s ago 22m 50.7M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (34s) 28s ago 21m 54.0M - 3.5 e1d6a67b021e 35ba1bf32537 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (108s) 2s ago 27m 452M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (11m) 28s ago 28m 506M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (29m) 28s ago 29m 78.7M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (27m) 2s ago 27m 56.8M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (27m) 28s ago 27m 56.4M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (89s) 28s ago 23m 13.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (83s) 2s ago 23m 15.2M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (27m) 28s ago 27m 55.8M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (26m) 28s ago 26m 55.3M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:55:18.108 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (26m) 28s ago 26m 53.4M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:55:18.109 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (25m) 28s ago 25m 53.5M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:55:18.109 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (25m) 2s ago 25m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:55:18.109 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (25m) 2s ago 25m 52.4M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:55:18.109 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (24m) 2s ago 24m 52.7M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:55:18.109 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (24m) 2s ago 24m 53.1M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:55:18.109 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (57s) 2s ago 23m 51.8M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:55:18.109 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (22m) 28s ago 22m 75.2M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:55:18.109 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (21m) 2s ago 21m 79.1M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:55:18.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:17 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:18.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:17 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:18.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:17 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:18.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:17 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:17 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:17 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:18.519 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:55:18.519 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:55:18.519 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:55:18.519 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:55:18.520 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:55:18.521 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:55:18.521 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:55:18.867 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:55:18.867 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:55:18.867 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:55:18.867 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-20T22:55:18.867 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [ 2024-05-20T22:55:18.867 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr" 2024-05-20T22:55:18.868 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:55:18.868 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "2/2 daemons upgraded", 2024-05-20T22:55:18.868 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading grafana daemons", 2024-05-20T22:55:18.868 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:55:18.868 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:55:19.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:18 smithi138 ceph-mon[25673]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:19.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:18 smithi138 ceph-mon[25673]: from='client.15720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:19.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:18 smithi138 ceph-mon[25673]: from='client.15726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:19.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:18 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/724858011' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:19.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:18 smithi055 ceph-mon[24757]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:19.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:18 smithi055 ceph-mon[24757]: from='client.15720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:19.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:18 smithi055 ceph-mon[24757]: from='client.15726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:19.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:18 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/724858011' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:19.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:18 smithi055 ceph-mon[29498]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:19.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:18 smithi055 ceph-mon[29498]: from='client.15720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:19.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:18 smithi055 ceph-mon[29498]: from='client.15726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:19.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:18 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/724858011' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:20.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:19 smithi055 ceph-mon[24757]: from='client.25621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:20.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:19 smithi055 ceph-mon[24757]: from='client.15744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:20.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:19 smithi055 ceph-mon[29498]: from='client.25621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:20.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:19 smithi055 ceph-mon[29498]: from='client.15744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:20.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:19 smithi138 ceph-mon[25673]: from='client.25621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:20.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:19 smithi138 ceph-mon[25673]: from='client.15744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:21.289 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[24757]: from='client.15705 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:21.289 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[24757]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:21.289 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[29498]: from='client.15705 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[29498]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:21.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:21.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:21.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:55:21.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:21.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:55:21.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:20 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:21.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:20 smithi138 ceph-mon[25673]: from='client.15705 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:21.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:20 smithi138 ceph-mon[25673]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:21.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:20 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:21.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:20 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:21.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:20 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:21.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:20 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:55:21.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:20 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:21.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:20 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:55:21.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:20 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:22.302 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:22 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:55:22.302 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:22 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:55:22.302 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:22 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:55:22.302 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:22 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:55:22.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:22 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:55:22.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:22 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:55:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:23 smithi055 ceph-mon[24757]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:23.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:23 smithi055 ceph-mon[29498]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:23.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:23 smithi138 ceph-mon[25673]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:23.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:55:23.598+0000 7f35b53ff700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:55:23.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:55:23.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:55:23.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:55:23.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:55:23.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:23.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:23.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:55:23.984 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:55:23.985 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:55:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:55:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:55:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:55:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:55:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:55:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:55:23.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return future.result(timeout) 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return self.__get_result() 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: raise self._exception 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: use_current_daemon_image=reconfig, 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:55:23.987 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Creating ceph-iscsi config... 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: Traceback (most recent call last): 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: return _run_code(code, main_globals, None, 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: exec(code, run_globals) 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:55:23.988 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:55:23.989 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:55:23.989 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:55:23.989 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:55:23.989 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:23.989 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:55:24.357 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:55:24.358 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: return future.result(timeout) 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: return self.__get_result() 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: raise self._exception 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:55:24.359 INFO:journalctl@ceph.mon.b.smithi138.stdout: Creating ceph-iscsi config... 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: Traceback (most recent call last): 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: exec(code, run_globals) 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:55:24.360 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: Upgrade: Finalizing container_image settings 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:55:24.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:55:24.363 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: Upgrade: Complete! 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:55:24.364 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:24.365 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:24.365 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:55:24.365 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:24.365 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:55:24.365 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:55:24.365 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:24.365 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:24 smithi138 ceph-mon[25673]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:55:24.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:24.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:55:24.483 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:55:24.483 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:55:24.484 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: return future.result(timeout) 2024-05-20T22:55:24.485 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: return self.__get_result() 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: raise self._exception 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:55:24.486 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:55:24.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:55:24.488 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-20T22:55:24.489 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: return future.result(timeout) 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: return self.__get_result() 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: raise self._exception 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_current_daemon_image=reconfig, 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-20T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi055.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: Creating ceph-iscsi config... 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: Non-zero exit code 1 from systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: Traceback (most recent call last): 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: exec(code, run_globals) 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T22:55:24.491 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T22:55:24.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T22:55:24.492 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T22:55:24.492 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:55:24.492 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:55:24.492 INFO:journalctl@ceph.mon.a.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:55:24.492 INFO:journalctl@ceph.mon.a.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.492 INFO:journalctl@ceph.mon.a.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:55:24.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: Upgrade: Finalizing container_image settings 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1123, in deploy_daemon 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.c.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.c.smithi055.stdout: RuntimeError: Failed command: systemctl restart ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T22:55:24.494 INFO:journalctl@ceph.mon.c.smithi055.stdout: See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: Upgrade: Finalizing container_image settings 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:55:24.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:55:24.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:55:24.498 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:55:24.499 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:55:24.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:55:24.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.500 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.500 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.500 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.500 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: Upgrade: Complete! 2024-05-20T22:55:24.500 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[24757]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.501 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: Upgrade: Complete! 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: Reconfiguring iscsi.foo.smithi055.akoeuh (dependencies changed)... 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:24.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:24 smithi055 ceph-mon[29498]: Reconfiguring daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T22:55:25.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:25 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:25.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:25 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:25 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:25 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:25 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:25 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:26.225 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:26 smithi055 ceph-mon[24757]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T22:55:26.226 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:26 smithi055 ceph-mon[29498]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T22:55:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:26 smithi138 ceph-mon[25673]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T22:55:27.521 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:55:27] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.43.0" 2024-05-20T22:55:27.522 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:27.522 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:27.522 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:55:27.522 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[24757]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:55:27.522 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:55:27.522 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[24757]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:55:27.522 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:27.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3947818124' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:55:27.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:27.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:27.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:55:27.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[29498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:55:27.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:55:27.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[29498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:55:27.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:27.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:27 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3947818124' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:55:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:55:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:27 smithi138 ceph-mon[25673]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-20T22:55:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:55:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:27 smithi138 ceph-mon[25673]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-20T22:55:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:27 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:55:27.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:27 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3947818124' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T22:55:28.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:28 smithi138 ceph-mon[25673]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T22:55:28.967 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:28 smithi055 ceph-mon[24757]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T22:55:28.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:28 smithi055 ceph-mon[29498]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T22:55:30.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:30 smithi055 ceph-mon[24757]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T22:55:30.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:30 smithi055 ceph-mon[29498]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T22:55:30.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:30 smithi138 ceph-mon[25673]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T22:55:32.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:31 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:32.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:31 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:32.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:31 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:32.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:31 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:55:32.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:31 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:55:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:32.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:32.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:32.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:55:32.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:55:32.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:31 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:55:33.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:32 smithi138 ceph-mon[25673]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:33.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:32 smithi055 ceph-mon[24757]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:33.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:32 smithi055 ceph-mon[29498]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:34 smithi138 ceph-mon[25673]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:35.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:34 smithi055 ceph-mon[24757]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:34 smithi055 ceph-mon[29498]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-20T22:55:36.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:36 smithi055 ceph-mon[24757]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:36.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:36 smithi055 ceph-mon[29498]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:36 smithi138 ceph-mon[25673]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-20T22:55:37.536 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:55:37] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.43.0" 2024-05-20T22:55:38.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:37 smithi138 ceph-mon[25673]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:38.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:37 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:38.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:37 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:39.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:38 smithi138 ceph-mon[25673]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:55:39.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:38 smithi055 ceph-mon[24757]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:55:39.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:38 smithi055 ceph-mon[29498]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:55:40.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:39 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:40.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:39 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:40.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:39 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:41.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:40 smithi138 ceph-mon[25673]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:55:41.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:40 smithi055 ceph-mon[24757]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:55:41.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:40 smithi055 ceph-mon[29498]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-20T22:55:43.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:42 smithi138 ceph-mon[25673]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:43.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:42 smithi055 ceph-mon[24757]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:43.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:42 smithi055 ceph-mon[29498]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:45.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:44 smithi055 ceph-mon[24757]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:45.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:44 smithi055 ceph-mon[29498]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:45.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:44 smithi138 ceph-mon[25673]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:46.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:46 smithi055 ceph-mon[24757]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:46.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:46 smithi055 ceph-mon[29498]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:47.158 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:46 smithi138 ceph-mon[25673]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:47.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:55:47] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.43.0" 2024-05-20T22:55:48.177 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:47 smithi138 ceph-mon[25673]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:48.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:47 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:47 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:49.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:48 smithi055 ceph-mon[24757]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:49.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:48 smithi055 ceph-mon[29498]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:49.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:48 smithi138 ceph-mon[25673]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:49.837 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-05-20T22:55:51.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:50 smithi055 ceph-mon[24757]: from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:51.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:50 smithi055 ceph-mon[24757]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:51.137 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:50 smithi055 ceph-mon[29498]: from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:51.138 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:50 smithi055 ceph-mon[29498]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:51.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:50 smithi138 ceph-mon[25673]: from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:55:51.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:50 smithi138 ceph-mon[25673]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:51.900 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:55:52.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:51 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3428438394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:52.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:51 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3428438394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:52.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:51 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3428438394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:52.462 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-05-20T22:55:53.106 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:52 smithi055 ceph-mon[24757]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:53.107 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:52 smithi055 ceph-mon[29498]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:53.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:52 smithi138 ceph-mon[25673]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:54.453 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)" 2024-05-20T22:55:54.942 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-05-20T22:55:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:54 smithi055 ceph-mon[24757]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:54 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/214759865' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:55.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:54 smithi055 ceph-mon[29498]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:55.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:55.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:54 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/214759865' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:55.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:54 smithi138 ceph-mon[25673]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:55.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:54 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:55:55.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:54 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/214759865' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:56.884 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:55:57.200 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:56 smithi055 ceph-mon[24757]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:57.200 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:56 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2217056193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:57.200 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:56 smithi055 ceph-mon[29498]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:57.200 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:56 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2217056193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:57.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:56 smithi138 ceph-mon[25673]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:55:57.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:56 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/2217056193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:55:57.458 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:55:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:55:57] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.43.0" 2024-05-20T22:55:57.482 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-05-20T22:55:58.046 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:57 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:58.047 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:57 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:57 smithi138 ceph-mon[25673]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:55:59.085 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:55:58 smithi055 ceph-mon[24757]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:59.086 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:55:58 smithi055 ceph-mon[29498]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:55:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:55:58 smithi138 ceph-mon[25673]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:01.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:00 smithi138 ceph-mon[25673]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:01.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:00 smithi138 ceph-mon[25673]: from='client.25687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:01.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:00 smithi055 ceph-mon[24757]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:01.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:00 smithi055 ceph-mon[24757]: from='client.25687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:01.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:00 smithi055 ceph-mon[29498]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:01.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:00 smithi055 ceph-mon[29498]: from='client.25687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:01.565 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:56:02.131 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade status' 2024-05-20T22:56:03.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:03 smithi055 ceph-mon[24757]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:03.116 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:03 smithi055 ceph-mon[29498]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:03.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:03 smithi138 ceph-mon[25673]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:04.034 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:56:04.035 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": null, 2024-05-20T22:56:04.035 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": false, 2024-05-20T22:56:04.035 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "", 2024-05-20T22:56:04.035 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T22:56:04.035 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": null, 2024-05-20T22:56:04.035 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "", 2024-05-20T22:56:04.035 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:56:04.035 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:56:04.608 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T22:56:05.203 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:05 smithi055 ceph-mon[24757]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:05.203 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:05 smithi055 ceph-mon[29498]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:05 smithi138 ceph-mon[25673]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:06.048 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:06 smithi055 ceph-mon[24757]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:06.048 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:06 smithi055 ceph-mon[24757]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:06.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:06 smithi055 ceph-mon[29498]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:06.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:06 smithi055 ceph-mon[29498]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:06.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:06 smithi138 ceph-mon[25673]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:06.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:06 smithi138 ceph-mon[25673]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:06.563 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T22:56:07.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:07 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/414052828' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:56:07.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:07 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:07.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:07 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/414052828' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:56:07.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:07 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:07.208 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-05-20T22:56:07.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:07 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/414052828' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:56:07.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:07 smithi138 ceph-mon[25673]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:07.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:56:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:56:07] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.43.0" 2024-05-20T22:56:08.723 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:08 smithi055 ceph-mon[24757]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:08.723 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:08 smithi055 ceph-mon[29498]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:08 smithi138 ceph-mon[25673]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:09 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:09.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:09 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:09.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:09 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:10.702 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:10 smithi055 ceph-mon[24757]: from='client.15816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:10.702 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:10 smithi055 ceph-mon[24757]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:10.702 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:10 smithi055 ceph-mon[24757]: from='client.15822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "mon", "hosts": "smithi138", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:10.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:10 smithi055 ceph-mon[29498]: from='client.15816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:10.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:10 smithi055 ceph-mon[29498]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:10.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:10 smithi055 ceph-mon[29498]: from='client.15822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "mon", "hosts": "smithi138", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:10.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:10 smithi138 ceph-mon[25673]: from='client.15816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:10.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:10 smithi138 ceph-mon[25673]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:10.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:10 smithi138 ceph-mon[25673]: from='client.15822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "mon", "hosts": "smithi138", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:11.552 INFO:teuthology.orchestra.run.smithi055.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:56:12.146 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-20T22:56:12.750 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[24757]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:12.750 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[24757]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:56:12.750 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:12.750 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:12.750 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:12.750 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:12.750 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:12.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[29498]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:12.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[29498]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:56:12.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:12.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:12.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:12.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:12.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:12.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:12 smithi138 ceph-mon[25673]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:12.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:12 smithi138 ceph-mon[25673]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:56:12.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:12 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:12.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:12 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:12.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:12 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:12.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:12 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:12.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:12 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:13.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:13 smithi138 ceph-mon[25673]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:56:13.876 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:13 smithi055 ceph-mon[24757]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:56:13.876 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:13 smithi055 ceph-mon[29498]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:56:14.605 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:56:14.715 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:14 smithi055 ceph-mon[24757]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:14.716 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:14 smithi055 ceph-mon[29498]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:14.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:14 smithi138 ceph-mon[25673]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:15.291 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:56:15.291 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (90s) 44s ago 23m 26.6M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T22:56:15.291 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (64s) 59s ago 23m 50.7M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (49s) 44s ago 22m 45.9M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (2m) 59s ago 28m 452M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (12m) 44s ago 29m 514M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (29m) 44s ago 30m 79.7M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (28m) 59s ago 28m 56.8M 2048M 17.2.0 e1d6a67b021e 2cd8871f6dc7 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (28m) 44s ago 28m 58.4M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (2m) 44s ago 24m 15.9M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (2m) 59s ago 24m 15.2M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (27m) 44s ago 27m 55.8M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (27m) 44s ago 27m 55.3M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (27m) 44s ago 27m 53.4M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (26m) 44s ago 26m 53.5M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:56:15.292 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (26m) 59s ago 26m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:56:15.293 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (26m) 59s ago 25m 52.4M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:56:15.293 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (25m) 59s ago 25m 52.7M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:56:15.293 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (25m) 59s ago 25m 53.1M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:56:15.293 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (114s) 59s ago 24m 51.8M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:56:15.293 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (22m) 44s ago 22m 75.4M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:56:15.293 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (22m) 59s ago 22m 79.1M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:56:15.708 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:56:15.708 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:56:15.708 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-20T22:56:15.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:56:15.708 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:56:15.709 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:56:15.710 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:56:15.710 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[24757]: from='client.15828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:15.710 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[24757]: from='client.25723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:15.710 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:15.710 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:15.710 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:15.710 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:15.710 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:56:15.710 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-20T22:56:15.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[29498]: from='client.15828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:15.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[29498]: from='client.25723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:15.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:15.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:15.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:15.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:15.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:56:15.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:15 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-20T22:56:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:15 smithi138 ceph-mon[25673]: from='client.15828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:15 smithi138 ceph-mon[25673]: from='client.25723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:15 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:15 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:15 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:15 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:15 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:56:15.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:15 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-20T22:56:16.056 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:56:16.057 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:56:16.057 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:56:16.057 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi138", 2024-05-20T22:56:16.057 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T22:56:16.057 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "0/1 daemons upgraded", 2024-05-20T22:56:16.057 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading mon daemons", 2024-05-20T22:56:16.057 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:56:16.057 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:56:16.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:16 smithi138 ceph-mon[25673]: from='client.25729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:16 smithi138 ceph-mon[25673]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:16 smithi138 ceph-mon[25673]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:56:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:16 smithi138 ceph-mon[25673]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:56:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:16 smithi138 ceph-mon[25673]: Upgrade: Setting container_image for all mgr 2024-05-20T22:56:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:16 smithi138 ceph-mon[25673]: Upgrade: It appears safe to stop mon.b 2024-05-20T22:56:16.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:16 smithi138 ceph-mon[25673]: from='client.? 172.21.15.55:0/3494762490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[24757]: from='client.25729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[24757]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[24757]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:56:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[24757]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:56:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all mgr 2024-05-20T22:56:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[24757]: Upgrade: It appears safe to stop mon.b 2024-05-20T22:56:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/3494762490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:16.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[29498]: from='client.25729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[29498]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[29498]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:56:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[29498]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:56:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all mgr 2024-05-20T22:56:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[29498]: Upgrade: It appears safe to stop mon.b 2024-05-20T22:56:16.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:16 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/3494762490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:17.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:56:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:56:17] "GET /metrics HTTP/1.1" 200 37918 "" "Prometheus/2.43.0" 2024-05-20T22:56:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:18 smithi138 ceph-mon[25673]: from='client.15852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:18 smithi138 ceph-mon[25673]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:18 smithi138 ceph-mon[25673]: Upgrade: Updating mon.b 2024-05-20T22:56:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:18 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:18 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:56:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:18 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:56:18.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:18 smithi138 ceph-mon[25673]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:18.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:18 smithi138 ceph-mon[25673]: Deploying daemon mon.b on smithi138 2024-05-20T22:56:18.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[29498]: from='client.15852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:18.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:18.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[29498]: Upgrade: Updating mon.b 2024-05-20T22:56:18.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:18.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:56:18.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[29498]: Deploying daemon mon.b on smithi138 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[24757]: from='client.15852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[24757]: Upgrade: Updating mon.b 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:18.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:18 smithi055 ceph-mon[24757]: Deploying daemon mon.b on smithi138 2024-05-20T22:56:19.173 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:18 smithi138 systemd[1]: Stopping Ceph mon.b for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:56:19.173 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 ceph-mon[25673]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:19.173 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b[25669]: 2024-05-20T22:56:19.149+0000 7fd4ebf80700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:56:19.173 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b[25669]: 2024-05-20T22:56:19.149+0000 7fd4ebf80700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-05-20T22:56:19.434 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 podman[67652]: 2024-05-20 22:56:19.181421945 +0000 UTC m=+0.125200961 container died 2cd8871f6dc7db4d547ff9f99b05f34cdc1a4ff21b047f01ed3769341989517d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.expose-services=, vendor=Red Hat, Inc., name=centos-stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, ceph=True, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754) 2024-05-20T22:56:19.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:19 smithi055 ceph-mon[24757]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:19.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:19 smithi055 ceph-mon[29498]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:19.836 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 podman[67652]: 2024-05-20 22:56:19.534550569 +0000 UTC m=+0.478329569 container cleanup 2cd8871f6dc7db4d547ff9f99b05f34cdc1a4ff21b047f01ed3769341989517d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, release=754, vendor=Red Hat, Inc., GIT_CLEAN=True, distribution-scope=public, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-20T22:56:19.836 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 bash[67652]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b 2024-05-20T22:56:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 podman[67664]: 2024-05-20 22:56:19.844490012 +0000 UTC m=+0.660326254 container remove 2cd8871f6dc7db4d547ff9f99b05f34cdc1a4ff21b047f01ed3769341989517d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b, GIT_BRANCH=HEAD, release=754, GIT_CLEAN=True, version=8, ceph=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-20T22:56:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.b.service: Deactivated successfully. 2024-05-20T22:56:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 systemd[1]: Stopped Ceph mon.b for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:56:20.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:19 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.b.service: Consumed 19.416s CPU time. 2024-05-20T22:56:20.394 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 systemd[1]: Starting Ceph mon.b for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:56:20.648 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 podman[67804]: 2024-05-20 22:56:20.305888595 +0000 UTC m=+0.017472053 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:56:20.648 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 podman[67804]: 2024-05-20 22:56:20.420703715 +0000 UTC m=+0.132287144 container create e6d39c7bef115831bb5e8b8b6adcffb0cebffdd7dfb35ae08d7f778e5dc79a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:56:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 podman[67804]: 2024-05-20 22:56:20.657136421 +0000 UTC m=+0.368719875 container init e6d39c7bef115831bb5e8b8b6adcffb0cebffdd7dfb35ae08d7f778e5dc79a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, io.buildah.version=1.33.5) 2024-05-20T22:56:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 podman[67804]: 2024-05-20 22:56:20.660303841 +0000 UTC m=+0.371887292 container start e6d39c7bef115831bb5e8b8b6adcffb0cebffdd7dfb35ae08d7f778e5dc79a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-20T22:56:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T22:56:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev), process ceph-mon, pid 2 2024-05-20T22:56:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: pidfile_write: ignore empty --pid-file 2024-05-20T22:56:21.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: load: jerasure load: lrc 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: RocksDB version: 7.9.2 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Git sha 0 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Compile date 2024-05-17 12:39:40 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: DB SUMMARY 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: DB Session ID: JF08FKW2MSPPH8GP01IO 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: CURRENT file: CURRENT 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2828 Bytes 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 3486107 ; 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.error_if_exists: 0 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.create_if_missing: 0 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.paranoid_checks: 1 2024-05-20T22:56:21.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-20T22:56:21.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T22:56:21.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-20T22:56:21.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.env: 0x55afb9a85c60 2024-05-20T22:56:21.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.fs: PosixFileSystem 2024-05-20T22:56:21.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.info_log: 0x55afbb881ac0 2024-05-20T22:56:21.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T22:56:21.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.statistics: (nil) 2024-05-20T22:56:21.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.use_fsync: 0 2024-05-20T22:56:21.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_log_file_size: 0 2024-05-20T22:56:21.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.allow_fallocate: 1 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.use_direct_reads: 0 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.db_log_dir: 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.wal_dir: 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T22:56:21.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.write_buffer_manager: 0x55afbb854820 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T22:56:21.111 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.unordered_write: 0 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.row_cache: None 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.wal_filter: None 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.two_write_queues: 0 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.wal_compression: 0 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.atomic_flush: 0 2024-05-20T22:56:21.112 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.log_readahead_size: 0 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_background_jobs: 2 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_background_compactions: -1 2024-05-20T22:56:21.113 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_subcompactions: 1 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_open_files: -1 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T22:56:21.114 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_background_flushes: -1 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Compression algorithms supported: 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: kZSTD supported: 0 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: kXpressCompression supported: 0 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: kZlibCompression supported: 1 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: kSnappyCompression supported: 1 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: kLZ4Compression supported: 1 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: kBZip2Compression supported: 0 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-20T22:56:21.115 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.merge_operator: 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_filter: None 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55afbb881bc0) 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout: cache_index_and_filter_blocks: 1 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T22:56:21.116 INFO:journalctl@ceph.mon.b.smithi138.stdout: pin_top_level_index_and_filter: 1 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: index_type: 0 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: data_block_index_type: 0 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: index_shortening: 1 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: checksum: 4 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: no_block_cache: 0 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache: 0x55afbb885090 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache_name: BinnedLRUCache 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache_options: 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: capacity : 536870912 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: num_shard_bits : 4 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: strict_capacity_limit : 0 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: high_pri_pool_ratio: 0.000 2024-05-20T22:56:21.117 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_cache_compressed: (nil) 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: persistent_cache: (nil) 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_size: 4096 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_size_deviation: 10 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_restart_interval: 16 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: index_block_restart_interval: 1 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: metadata_block_size: 4096 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: partition_filters: 0 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: use_delta_encoding: 1 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: filter_policy: bloomfilter 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: whole_key_filtering: 1 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: verify_compression: 0 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: read_amp_bytes_per_bit: 0 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: format_version: 5 2024-05-20T22:56:21.118 INFO:journalctl@ceph.mon.b.smithi138.stdout: enable_index_compression: 1 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout: block_align: 0 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout: max_auto_readahead_size: 262144 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout: prepopulate_block_cache: 0 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout: initial_auto_readahead_size: 8192 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout: num_file_reads_for_auto_readahead: 2 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression: NoCompression 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.num_levels: 7 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T22:56:21.119 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T22:56:21.120 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T22:56:21.121 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.arena_block_size: 1048576 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T22:56:21.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.table_properties_collectors: 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.inplace_update_support: 0 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T22:56:21.123 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T22:56:21.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T22:56:21.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T22:56:21.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.bloom_locality: 0 2024-05-20T22:56:21.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.max_successive_merges: 0 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.ttl: 2592000 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.enable_blob_files: false 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.min_blob_size: 0 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T22:56:21.126 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.blob_file_starting_level: 0 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28871, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 14eeae4c-8f10-40a0-b41e-7b78ae834ddf 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716245780690860, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-05-20T22:56:21.127 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716245780702585, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 2136816, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28876, "largest_seqno": 30901, "table_properties": {"data_size": 2129230, "index_size": 4710, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1861, "raw_key_size": 17358, "raw_average_key_size": 23, "raw_value_size": 2113572, "raw_average_value_size": 2848, "num_data_blocks": 213, "num_entries": 742, "num_filter_entries": 742, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716245780, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "14eeae4c-8f10-40a0-b41e-7b78ae834ddf", "db_session_id": "JF08FKW2MSPPH8GP01IO", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716245780702681, "job": 1, "event": "recovery_finished"} 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55afbb976000 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: DB pointer 0x55afbb960000 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** DB Stats ** 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:56:21.128 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** Compaction Stats [default] ** 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: L0 1/0 2.04 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 471.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: L6 1/0 8.67 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: Sum 2/0 10.71 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 471.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 471.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** Compaction Stats [default] ** 2024-05-20T22:56:21.129 INFO:journalctl@ceph.mon.b.smithi138.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 471.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: Cumulative compaction: 0.00 GB write, 141.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:56:21.130 INFO:journalctl@ceph.mon.b.smithi138.stdout: Interval compaction: 0.00 GB write, 141.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout: Block cache BinnedLRUCache@0x55afbb885090#2 capacity: 512.00 MB usage: 7.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: starting mon.b rank 2 at public addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] at bind addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: mon.b@-1(???) e3 preinit fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: mon.b@-1(???).mds e1 new map 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: mon.b@-1(???).mds e1 print_map 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout: e1 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:56:21.131 INFO:journalctl@ceph.mon.b.smithi138.stdout: legacy client fscid: -1 2024-05-20T22:56:21.132 INFO:journalctl@ceph.mon.b.smithi138.stdout: 2024-05-20T22:56:21.132 INFO:journalctl@ceph.mon.b.smithi138.stdout: No filesystems configured 2024-05-20T22:56:21.132 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: mon.b@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-20T22:56:21.132 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:56:21.132 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:56:21.132 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:56:21.132 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 ceph-mon[67823]: mon.b@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-05-20T22:56:21.132 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 bash[67804]: e6d39c7bef115831bb5e8b8b6adcffb0cebffdd7dfb35ae08d7f778e5dc79a10 2024-05-20T22:56:21.132 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:20 smithi138 systemd[1]: Started Ceph mon.b for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:56:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: mon.b calling monitor election 2024-05-20T22:56:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: mon.a calling monitor election 2024-05-20T22:56:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:56:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:56:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: fsmap 2024-05-20T22:56:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:56:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: mgrmap e42: y(active, since 3m), standbys: x 2024-05-20T22:56:21.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: overall HEALTH_OK 2024-05-20T22:56:21.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:21.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:21.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:21 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:22.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: mon.b calling monitor election 2024-05-20T22:56:22.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: mon.a calling monitor election 2024-05-20T22:56:22.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:56:22.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: fsmap 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: mgrmap e42: y(active, since 3m), standbys: x 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: overall HEALTH_OK 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: mon.b calling monitor election 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: mon.a calling monitor election 2024-05-20T22:56:22.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:56:22.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} 2024-05-20T22:56:22.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: fsmap 2024-05-20T22:56:22.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:56:22.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: mgrmap e42: y(active, since 3m), standbys: x 2024-05-20T22:56:22.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: overall HEALTH_OK 2024-05-20T22:56:22.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:22.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:22.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:21 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:23.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:22 smithi138 ceph-mon[67823]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:22 smithi055 ceph-mon[24757]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:23.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:22 smithi055 ceph-mon[29498]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:24.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:24 smithi138 ceph-mon[67823]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:24.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:24 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:25.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:24 smithi055 ceph-mon[29498]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:25.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:24 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:24 smithi055 ceph-mon[24757]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:24 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:25.986 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:56:25 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:56:25.711+0000 7f35b8405700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-05-20T22:56:26.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:26 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:26.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:26 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:26.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:26 smithi055 ceph-mon[24757]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:26.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:26 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:26.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:26 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:26.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:26 smithi055 ceph-mon[29498]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:26 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:26 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:26 smithi138 ceph-mon[67823]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:27.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:27 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:27.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:27 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:27.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:27 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:27.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:27.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:27 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:27.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:27 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:27.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:27.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:27 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:27.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:27 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:27.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:56:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:56:27] "GET /metrics HTTP/1.1" 200 37859 "" "Prometheus/2.43.0" 2024-05-20T22:56:28.832 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:28 smithi138 ceph-mon[67823]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:28.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:28 smithi055 ceph-mon[24757]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:28.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:28 smithi055 ceph-mon[29498]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:30.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Detected new or changed devices on smithi138 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all crash 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mds 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nfs 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all node-exporter 2024-05-20T22:56:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all prometheus 2024-05-20T22:56:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all alertmanager 2024-05-20T22:56:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.860 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all grafana 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all loki 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all promtail 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Finalizing container_image settings 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:56:30.861 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:56:30.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:56:30.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:56:30.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:56:30.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:56:30.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:56:30.862 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.863 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: Upgrade: Complete! 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:30.864 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:30.865 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:30 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Detected new or changed devices on smithi138 2024-05-20T22:56:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all crash 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all mds 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all nfs 2024-05-20T22:56:30.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:56:30.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:30.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Detected new or changed devices on smithi138 2024-05-20T22:56:30.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all crash 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all mds 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all nfs 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all node-exporter 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all prometheus 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all alertmanager 2024-05-20T22:56:30.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all grafana 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all loki 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all promtail 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Finalizing container_image settings 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all node-exporter 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all prometheus 2024-05-20T22:56:30.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all alertmanager 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all grafana 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all loki 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all promtail 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Finalizing container_image settings 2024-05-20T22:56:30.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:56:30.993 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.994 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: Upgrade: Complete! 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:56:30.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: Upgrade: Complete! 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:30.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.998 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:30.998 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:56:30.998 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:56:30.998 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:56:30.998 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:30 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:32.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:32 smithi138 ceph-mon[67823]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:32.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:32 smithi055 ceph-mon[29498]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:32.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:32 smithi055 ceph-mon[24757]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:34.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:34 smithi138 ceph-mon[67823]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:34.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:34 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:34.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:34 smithi055 ceph-mon[29498]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:34.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:34 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:34.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:34 smithi055 ceph-mon[24757]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:34.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:34 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:56:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:36 smithi138 ceph-mon[67823]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:36.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:36 smithi055 ceph-mon[24757]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:36.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:36 smithi055 ceph-mon[29498]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:37.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:56:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:56:37] "GET /metrics HTTP/1.1" 200 37859 "" "Prometheus/2.43.0" 2024-05-20T22:56:37.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:37 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:37.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:37 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:37.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:37 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:38.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:38 smithi138 ceph-mon[67823]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:38.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:38 smithi055 ceph-mon[24757]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:38.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:38 smithi055 ceph-mon[29498]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:39.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:39 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:39.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:39 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:39.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:39 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:40 smithi138 ceph-mon[67823]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:40.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:40 smithi055 ceph-mon[24757]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:40.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:40 smithi055 ceph-mon[29498]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:42.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:42 smithi138 ceph-mon[67823]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:42.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:42 smithi055 ceph-mon[24757]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:42.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:42 smithi055 ceph-mon[29498]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:44.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:44 smithi055 ceph-mon[24757]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:44.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:44 smithi055 ceph-mon[29498]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:45.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:44 smithi138 ceph-mon[67823]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:46.714 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:46 smithi055 ceph-mon[24757]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:46.714 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:46 smithi055 ceph-mon[29498]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:47.033 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T22:56:47.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:46 smithi138 ceph-mon[67823]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:47.349 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:56:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:56:47] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-05-20T22:56:47.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:47 smithi055 ceph-mon[29498]: from='client.15858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:47.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:47 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:47.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:47 smithi055 ceph-mon[24757]: from='client.15858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:47.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:47 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:48.007 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:47 smithi138 ceph-mon[67823]: from='client.15858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:48.007 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:47 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:48.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:48 smithi138 ceph-mon[67823]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:48.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:48 smithi055 ceph-mon[24757]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:48.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:48 smithi055 ceph-mon[29498]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (2m) 78s ago 24m 26.6M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (98s) 23s ago 23m 74.4M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (82s) 78s ago 23m 45.9M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (3m) 23s ago 28m 453M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (13m) 78s ago 30m 514M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (30m) 78s ago 30m 79.7M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (28s) 23s ago 29m 19.9M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (29m) 78s ago 29m 58.4M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:56:48.985 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (3m) 78s ago 24m 15.9M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (2m) 23s ago 24m 15.9M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (28m) 78s ago 28m 55.8M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (28m) 78s ago 28m 55.3M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (27m) 78s ago 27m 53.4M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (27m) 78s ago 27m 53.5M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (26m) 23s ago 26m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (26m) 23s ago 26m 52.4M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (26m) 23s ago 26m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (25m) 23s ago 25m 53.3M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (2m) 23s ago 25m 64.9M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (23m) 78s ago 23m 75.4M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:56:48.986 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (23m) 23s ago 23m 79.5M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:56:49.601 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-05-20T22:56:49.910 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:49 smithi055 ceph-mon[24757]: from='client.15864 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:49.910 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:49 smithi055 ceph-mon[29498]: from='client.15864 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:50.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:49 smithi138 ceph-mon[67823]: from='client.15864 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:50.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:50 smithi055 ceph-mon[24757]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:50.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:50 smithi055 ceph-mon[29498]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:51.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:50 smithi138 ceph-mon[67823]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:51.622 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:56:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:51 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2341843392' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:51.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:51 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2341843392' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:52.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:51 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2341843392' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:56:52.271 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade status' 2024-05-20T22:56:52.880 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:52 smithi055 ceph-mon[24757]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:52.881 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:52 smithi055 ceph-mon[29498]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:53.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:52 smithi138 ceph-mon[67823]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:54.169 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:56:54.169 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": null, 2024-05-20T22:56:54.169 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": false, 2024-05-20T22:56:54.169 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "", 2024-05-20T22:56:54.169 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T22:56:54.169 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": null, 2024-05-20T22:56:54.169 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "", 2024-05-20T22:56:54.169 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:56:54.169 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:56:54.727 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T22:56:54.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:54 smithi055 ceph-mon[24757]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:54.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:54 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:54.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:54 smithi055 ceph-mon[29498]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:54.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:54 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:55.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:54 smithi138 ceph-mon[67823]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:55.122 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:54 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:56:55.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:55 smithi055 ceph-mon[24757]: from='client.15876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:55.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:55 smithi055 ceph-mon[29498]: from='client.15876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:56.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:55 smithi138 ceph-mon[67823]: from='client.15876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:56:56.835 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T22:56:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:56 smithi055 ceph-mon[24757]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:56.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:56 smithi055 ceph-mon[29498]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:57.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:56 smithi138 ceph-mon[67823]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:56:57.394 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:56:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:56:57] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-05-20T22:56:57.395 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-05-20T22:56:57.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:57 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:57.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:57 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/2706620494' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:56:57.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:57 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:57.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:57 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/2706620494' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:56:58.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:57 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:56:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:57 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2706620494' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:56:58.890 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:56:58 smithi055 ceph-mon[24757]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:58.891 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:56:58 smithi055 ceph-mon[29498]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:56:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:56:58 smithi138 ceph-mon[67823]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:00.964 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:00 smithi055 ceph-mon[24757]: from='client.15888 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:00.964 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:00 smithi055 ceph-mon[24757]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:00.964 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:00 smithi055 ceph-mon[24757]: from='client.15894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "mon", "hosts": "smithi055", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:00.964 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:00 smithi055 ceph-mon[29498]: from='client.15888 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:00.964 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:00 smithi055 ceph-mon[29498]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:00.965 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:00 smithi055 ceph-mon[29498]: from='client.15894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "mon", "hosts": "smithi055", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:00 smithi138 ceph-mon[67823]: from='client.15888 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:00 smithi138 ceph-mon[67823]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:01.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:00 smithi138 ceph-mon[67823]: from='client.15894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "mon", "hosts": "smithi055", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:01.908 INFO:teuthology.orchestra.run.smithi055.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:57:02.542 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-20T22:57:03.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[24757]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:03.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[24757]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:57:03.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:03.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:03.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:03.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:57:03.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:03.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[29498]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:03.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[29498]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:57:03.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:03.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:03.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:03.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:57:03.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:02 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:02 smithi138 ceph-mon[67823]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:02 smithi138 ceph-mon[67823]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:57:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:02 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:02 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:02 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:02 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:57:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:02 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:04.218 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:03 smithi055 ceph-mon[24757]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:57:04.219 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:03 smithi055 ceph-mon[29498]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:57:04.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:03 smithi138 ceph-mon[67823]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:57:05.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:04 smithi055 ceph-mon[24757]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:05.123 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:04 smithi055 ceph-mon[29498]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:05.146 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:57:05.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:04 smithi138 ceph-mon[67823]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:05.832 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:57:05.832 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (2m) 95s ago 24m 26.6M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T22:57:05.832 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (114s) 40s ago 24m 74.4M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T22:57:05.832 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (99s) 95s ago 23m 45.9M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T22:57:05.832 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (3m) 40s ago 29m 453M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:57:05.832 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (13m) 95s ago 30m 514M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:57:05.832 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (30m) 95s ago 30m 79.7M 2048M 17.2.0 e1d6a67b021e 2a879cf14270 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (45s) 40s ago 29m 19.9M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (29m) 95s ago 29m 58.4M 2048M 17.2.0 e1d6a67b021e 79201364d4c8 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (3m) 95s ago 25m 15.9M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (3m) 40s ago 25m 15.9M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (28m) 95s ago 28m 55.8M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (28m) 95s ago 28m 55.3M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (28m) 95s ago 28m 53.4M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (27m) 95s ago 27m 53.5M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (27m) 40s ago 27m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (26m) 40s ago 26m 52.4M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (26m) 40s ago 26m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (26m) 40s ago 26m 53.3M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (2m) 40s ago 25m 64.9M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:57:05.833 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (23m) 95s ago 23m 75.4M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:57:05.834 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (23m) 40s ago 23m 79.5M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:57:06.269 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:57:06.269 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:57:06.269 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-20T22:57:06.269 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:57:06.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:57:06.269 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": {}, 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T22:57:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:57:06.271 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:57:06.628 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:57:06.628 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:57:06.628 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:57:06.629 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi055", 2024-05-20T22:57:06.629 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T22:57:06.629 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "0/2 daemons upgraded", 2024-05-20T22:57:06.629 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading mon daemons", 2024-05-20T22:57:06.629 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:57:06.629 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:57:06.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='client.15900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:06.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:06.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:06.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:57:06.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:57:06.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:06.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:06.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: Upgrade: Setting container_image for all mgr 2024-05-20T22:57:06.730 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='client.15906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:06.730 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:06.730 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:57:06.730 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-20T22:57:06.730 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: Upgrade: It appears safe to stop mon.a 2024-05-20T22:57:06.730 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='client.25801 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:06.730 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[24757]: from='client.? 172.21.15.55:0/509033565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:06.730 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='client.15900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:06.730 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: Upgrade: Setting container_image for all mgr 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='client.15906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-20T22:57:06.731 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: Upgrade: It appears safe to stop mon.a 2024-05-20T22:57:06.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='client.25801 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:06.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:06 smithi055 ceph-mon[29498]: from='client.? 172.21.15.55:0/509033565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='client.15900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:57:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:57:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mgr 2024-05-20T22:57:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='client.15906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:57:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-20T22:57:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: Upgrade: It appears safe to stop mon.a 2024-05-20T22:57:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='client.25801 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:06.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:06 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/509033565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:07.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:57:07] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-05-20T22:57:08.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[24757]: from='client.25813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:08.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[24757]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[24757]: Upgrade: Updating mon.a 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[24757]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[24757]: Deploying daemon mon.a on smithi055 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[29498]: from='client.25813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[29498]: Upgrade: Updating mon.a 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:08.216 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:08.217 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:08.217 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:08.217 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:07 smithi055 ceph-mon[29498]: Deploying daemon mon.a on smithi055 2024-05-20T22:57:08.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:07 smithi138 ceph-mon[67823]: from='client.25813 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:08.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:07 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:08.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:07 smithi138 ceph-mon[67823]: Upgrade: Updating mon.a 2024-05-20T22:57:08.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:07 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' 2024-05-20T22:57:08.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:07 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:08.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:07 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:08.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:07 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:08.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:07 smithi138 ceph-mon[67823]: Deploying daemon mon.a on smithi055 2024-05-20T22:57:09.063 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:08 smithi055 ceph-mon[29498]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:09.063 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:08 smithi055 systemd[1]: Stopping Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:57:09.063 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:08 smithi055 ceph-mon[24757]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:09.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:08 smithi138 ceph-mon[67823]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:09.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a[24753]: 2024-05-20T22:57:09.071+0000 7fb42540f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:57:09.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a[24753]: 2024-05-20T22:57:09.071+0000 7fb42540f700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-05-20T22:57:09.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:09 smithi055 podman[85945]: 2024-05-20 22:57:09.112069892 +0000 UTC m=+0.158684655 container died 2a879cf14270394e5475855fbfcb9d92158b49fd7b796de5db1d7ae6c79cc557 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, vendor=Red Hat, Inc., release=754, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container) 2024-05-20T22:57:09.398 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:09] ENGINE Bus STOPPING 2024-05-20T22:57:09.398 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T22:57:09.398 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:09] ENGINE Bus STOPPED 2024-05-20T22:57:09.398 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:09] ENGINE Bus STARTING 2024-05-20T22:57:09.398 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:09] ENGINE Serving on http://:::9283 2024-05-20T22:57:09.399 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:09 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:09] ENGINE Bus STARTED 2024-05-20T22:57:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:09 smithi055 podman[85945]: 2024-05-20 22:57:09.513471613 +0000 UTC m=+0.560086361 container cleanup 2a879cf14270394e5475855fbfcb9d92158b49fd7b796de5db1d7ae6c79cc557 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, name=centos-stream, version=8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, ceph=True, release=754, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T22:57:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:09 smithi055 bash[85945]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a 2024-05-20T22:57:10.159 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:09 smithi055 podman[85970]: 2024-05-20 22:57:09.773589923 +0000 UTC m=+0.654693180 container remove 2a879cf14270394e5475855fbfcb9d92158b49fd7b796de5db1d7ae6c79cc557 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, version=8, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-type=git, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , architecture=x86_64, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True) 2024-05-20T22:57:10.159 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:09 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.a.service: Deactivated successfully. 2024-05-20T22:57:10.159 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:09 smithi055 systemd[1]: Stopped Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:57:10.159 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:09 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.a.service: Consumed 29.161s CPU time. 2024-05-20T22:57:10.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 systemd[1]: Starting Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:57:10.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 podman[86098]: 2024-05-20 22:57:10.289024286 +0000 UTC m=+0.017955062 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:57:10.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 podman[86098]: 2024-05-20 22:57:10.399953899 +0000 UTC m=+0.128884655 container create 0581e8f15f4f3a909000b08bffebd230b0c3b65f4c2d78264f8e902265a9a323 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:57:10.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 podman[86098]: 2024-05-20 22:57:10.644497508 +0000 UTC m=+0.373428281 container init 0581e8f15f4f3a909000b08bffebd230b0c3b65f4c2d78264f8e902265a9a323 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-20T22:57:10.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 podman[86098]: 2024-05-20 22:57:10.647979917 +0000 UTC m=+0.376910681 container start 0581e8f15f4f3a909000b08bffebd230b0c3b65f4c2d78264f8e902265a9a323 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:57:10.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T22:57:10.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev), process ceph-mon, pid 2 2024-05-20T22:57:10.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: pidfile_write: ignore empty --pid-file 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: load: jerasure load: lrc 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: RocksDB version: 7.9.2 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Git sha 0 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Compile date 2024-05-17 12:39:40 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: DB SUMMARY 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: DB Session ID: HJNQHF86CNPX88OTMO8A 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: CURRENT file: CURRENT 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3094 Bytes 2024-05-20T22:57:10.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000063.sst 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000061.log size: 563070 ; 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.error_if_exists: 0 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.create_if_missing: 0 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.paranoid_checks: 1 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.env: 0x55e7a0ba5c60 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.fs: PosixFileSystem 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.info_log: 0x55e7a23c1ac0 2024-05-20T22:57:10.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.statistics: (nil) 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.use_fsync: 0 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_log_file_size: 0 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.allow_fallocate: 1 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T22:57:10.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.use_direct_reads: 0 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.db_log_dir: 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.wal_dir: 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T22:57:10.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.write_buffer_manager: 0x55e7a2394820 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T22:57:10.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T22:57:10.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T22:57:10.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.unordered_write: 0 2024-05-20T22:57:10.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T22:57:10.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T22:57:10.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T22:57:10.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T22:57:10.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.row_cache: None 2024-05-20T22:57:10.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.wal_filter: None 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.two_write_queues: 0 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.wal_compression: 0 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.atomic_flush: 0 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T22:57:10.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.log_readahead_size: 0 2024-05-20T22:57:10.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T22:57:10.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T22:57:10.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T22:57:10.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T22:57:10.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T22:57:10.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T22:57:10.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-20T22:57:10.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_background_jobs: 2 2024-05-20T22:57:10.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_background_compactions: -1 2024-05-20T22:57:10.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_subcompactions: 1 2024-05-20T22:57:10.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T22:57:10.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T22:57:10.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T22:57:10.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_open_files: -1 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T22:57:10.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_background_flushes: -1 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Compression algorithms supported: 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: kZSTD supported: 0 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: kXpressCompression supported: 0 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: kZlibCompression supported: 1 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: kSnappyCompression supported: 1 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: kLZ4Compression supported: 1 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: kBZip2Compression supported: 0 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T22:57:10.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.merge_operator: 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_filter: None 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e7a23c1bc0) 2024-05-20T22:57:10.997 INFO:journalctl@ceph.mon.a.smithi055.stdout: cache_index_and_filter_blocks: 1 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: pin_top_level_index_and_filter: 1 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_type: 0 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: data_block_index_type: 0 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_shortening: 1 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: checksum: 4 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: no_block_cache: 0 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache: 0x55e7a23c5090 2024-05-20T22:57:10.998 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_name: BinnedLRUCache 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_options: 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: capacity : 536870912 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: num_shard_bits : 4 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: strict_capacity_limit : 0 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: high_pri_pool_ratio: 0.000 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_compressed: (nil) 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: persistent_cache: (nil) 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_size: 4096 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_size_deviation: 10 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_restart_interval: 16 2024-05-20T22:57:10.999 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_block_restart_interval: 1 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: metadata_block_size: 4096 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: partition_filters: 0 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_delta_encoding: 1 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: filter_policy: bloomfilter 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: whole_key_filtering: 1 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: verify_compression: 0 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: read_amp_bytes_per_bit: 0 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: format_version: 5 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: enable_index_compression: 1 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_align: 0 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: max_auto_readahead_size: 262144 2024-05-20T22:57:11.000 INFO:journalctl@ceph.mon.a.smithi055.stdout: prepopulate_block_cache: 0 2024-05-20T22:57:11.001 INFO:journalctl@ceph.mon.a.smithi055.stdout: initial_auto_readahead_size: 8192 2024-05-20T22:57:11.001 INFO:journalctl@ceph.mon.a.smithi055.stdout: num_file_reads_for_auto_readahead: 2 2024-05-20T22:57:11.001 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T22:57:11.001 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T22:57:11.001 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression: NoCompression 2024-05-20T22:57:11.001 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T22:57:11.001 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T22:57:11.001 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T22:57:11.001 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.num_levels: 7 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T22:57:11.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:57:11.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:57:11.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T22:57:11.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T22:57:11.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T22:57:11.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T22:57:11.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:57:11.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:57:11.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T22:57:11.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T22:57:11.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.arena_block_size: 1048576 2024-05-20T22:57:11.005 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T22:57:11.006 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.table_properties_collectors: 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.inplace_update_support: 0 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.bloom_locality: 0 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.max_successive_merges: 0 2024-05-20T22:57:11.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.ttl: 2592000 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.enable_blob_files: false 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.min_blob_size: 0 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T22:57:11.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.blob_file_starting_level: 0 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 65, last_sequence is 29186, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-05-20T22:57:11.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 193ad7b1-5da5-4698-94d6-09a93e6f82dc 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716245830678669, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716245830681772, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 550495, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 29164, "largest_seqno": 29838, "table_properties": {"data_size": 547311, "index_size": 1338, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 837, "raw_key_size": 8461, "raw_average_key_size": 25, "raw_value_size": 540805, "raw_average_value_size": 1643, "num_data_blocks": 60, "num_entries": 329, "num_filter_entries": 329, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716245830, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "193ad7b1-5da5-4698-94d6-09a93e6f82dc", "db_session_id": "HJNQHF86CNPX88OTMO8A", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716245830681838, "job": 1, "event": "recovery_finished"} 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e7a24b6000 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: DB pointer 0x55e7a24a0000 2024-05-20T22:57:11.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** DB Stats ** 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:57:11.011 INFO:journalctl@ceph.mon.a.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: L0 1/0 537.59 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 277.8 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: L6 1/0 9.46 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: Sum 2/0 9.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 277.8 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 277.8 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 277.8 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-05-20T22:57:11.012 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative compaction: 0.00 GB write, 51.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval compaction: 0.00 GB write, 51.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:57:11.013 INFO:journalctl@ceph.mon.a.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:57:11.014 INFO:journalctl@ceph.mon.a.smithi055.stdout: Block cache BinnedLRUCache@0x55e7a23c5090#2 capacity: 512.00 MB usage: 2.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-05-20T22:57:11.014 INFO:journalctl@ceph.mon.a.smithi055.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-05-20T22:57:11.014 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:57:11.014 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:57:11.014 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: starting mon.a rank 0 at public addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] at bind addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:11.014 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: mon.a@-1(???) e3 preinit fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:11.014 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: mon.a@-1(???).mds e1 new map 2024-05-20T22:57:11.014 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: mon.a@-1(???).mds e1 print_map 2024-05-20T22:57:11.014 INFO:journalctl@ceph.mon.a.smithi055.stdout: e1 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout: legacy client fscid: -1 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout: No filesystems configured 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: mon.a@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 ceph-mon[86116]: mon.a@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-05-20T22:57:11.015 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 bash[86098]: 0581e8f15f4f3a909000b08bffebd230b0c3b65f4c2d78264f8e902265a9a323 2024-05-20T22:57:11.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:10 smithi055 systemd[1]: Started Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:57:12.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:12 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ignoring --setuser ceph since I am not root 2024-05-20T22:57:12.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:12 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ignoring --setgroup ceph since I am not root 2024-05-20T22:57:12.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mgr[63095]: -- 172.21.15.55:0/3404959594 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560abe07c5a0 con 0x560abe050c00 2024-05-20T22:57:12.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:12 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:12.325+0000 7f06b733f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:57:12.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:57:12.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: mon.a calling monitor election 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: monmap epoch 3 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: last_changed 2024-05-20T22:27:41.824528+0000 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: created 2024-05-20T22:26:07.295242+0000 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: min_mon_release 17 (quincy) 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: election_strategy: 1 2024-05-20T22:57:12.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: 0: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.a 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: 1: [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon.c 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: 2: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.b 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: fsmap 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: mgrmap e42: y(active, since 4m), standbys: x 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: overall HEALTH_OK 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: from='mgr.15336 ' entity='' 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[29498]: mgrmap e43: y(active, since 4m), standbys: x 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:57:12.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: mon.a calling monitor election 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: monmap epoch 3 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: last_changed 2024-05-20T22:27:41.824528+0000 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: created 2024-05-20T22:26:07.295242+0000 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: min_mon_release 17 (quincy) 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: election_strategy: 1 2024-05-20T22:57:12.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: 0: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.a 2024-05-20T22:57:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: 1: [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon.c 2024-05-20T22:57:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: 2: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.b 2024-05-20T22:57:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: fsmap 2024-05-20T22:57:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:57:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: mgrmap e42: y(active, since 4m), standbys: x 2024-05-20T22:57:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: overall HEALTH_OK 2024-05-20T22:57:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: from='mgr.15336 ' entity='' 2024-05-20T22:57:12.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:12 smithi055 ceph-mon[86116]: mgrmap e43: y(active, since 4m), standbys: x 2024-05-20T22:57:12.487 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: ignoring --setuser ceph since I am not root 2024-05-20T22:57:12.488 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: ignoring --setgroup ceph since I am not root 2024-05-20T22:57:12.488 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mgr[60799]: -- 172.21.15.138:0/80441048 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564c3e84c5a0 con 0x564c3e820c00 2024-05-20T22:57:12.488 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:12.308+0000 7f48d610a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T22:57:12.488 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:57:12.488 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:57:12.488 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:57:12.488 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:12.488 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: from='mgr.15336 172.21.15.55:0/1619118688' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:57:12.488 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: mon.a calling monitor election 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: monmap epoch 3 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: last_changed 2024-05-20T22:27:41.824528+0000 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: created 2024-05-20T22:26:07.295242+0000 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: min_mon_release 17 (quincy) 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: election_strategy: 1 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: 0: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.a 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: 1: [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon.c 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: 2: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.b 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: fsmap 2024-05-20T22:57:12.489 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:57:12.490 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: mgrmap e42: y(active, since 4m), standbys: x 2024-05-20T22:57:12.490 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: overall HEALTH_OK 2024-05-20T22:57:12.490 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: from='mgr.15336 ' entity='' 2024-05-20T22:57:12.490 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:12 smithi138 ceph-mon[67823]: mgrmap e43: y(active, since 4m), standbys: x 2024-05-20T22:57:12.855 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:12.494+0000 7f48d610a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:57:12.856 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:12.597+0000 7f48d610a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:57:12.947 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:12 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:12.520+0000 7f06b733f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:57:12.947 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:12 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:12.628+0000 7f06b733f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T22:57:12.947 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:12 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:12.955+0000 7f06b733f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:57:13.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:13.046+0000 7f06b733f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:57:13.355 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:12.902+0000 7f48d610a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T22:57:13.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:12.987+0000 7f48d610a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T22:57:14.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:14 smithi055 ceph-mon[86116]: mgrmap e44: y(active, since 4m), standbys: x 2024-05-20T22:57:14.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:14 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:14.313+0000 7f06b733f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:57:14.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:14 smithi055 ceph-mon[29498]: mgrmap e44: y(active, since 4m), standbys: x 2024-05-20T22:57:14.498 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:14 smithi138 ceph-mon[67823]: mgrmap e44: y(active, since 4m), standbys: x 2024-05-20T22:57:14.498 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:14.164+0000 7f48d610a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T22:57:14.499 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:14.334+0000 7f48d610a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:57:14.786 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:14.506+0000 7f48d610a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:57:14.971 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:14 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:14.494+0000 7f06b733f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T22:57:14.971 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:14 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:14.673+0000 7f06b733f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T22:57:15.080 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:14.793+0000 7f48d610a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:57:15.080 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:14.885+0000 7f48d610a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:57:15.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:14 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:14.979+0000 7f06b733f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T22:57:15.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:15 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:15.077+0000 7f06b733f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T22:57:15.355 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:15.088+0000 7f48d610a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:57:15.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:15.284+0000 7f48d610a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:57:15.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:15 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:15.291+0000 7f06b733f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T22:57:15.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:15 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:15.496+0000 7f06b733f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T22:57:16.299 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:15.887+0000 7f48d610a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:57:16.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:16 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:16.142+0000 7f06b733f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T22:57:16.602 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:16.307+0000 7f48d610a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:57:16.602 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:16.407+0000 7f48d610a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:57:16.602 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:16.609+0000 7f48d610a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T22:57:16.909 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:16 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:16.596+0000 7f06b733f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T22:57:16.910 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:16 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:16.700+0000 7f06b733f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T22:57:17.232 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:16 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:16.918+0000 7f06b733f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T22:57:17.465 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:17.131+0000 7f48d610a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:57:17.465 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:17.225+0000 7f48d610a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:57:17.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:17.474+0000 7f06b733f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T22:57:17.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:17.574+0000 7f06b733f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T22:57:17.815 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:17.473+0000 7f48d610a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:57:17.815 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:17.608+0000 7f48d610a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:57:17.815 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:17.705+0000 7f48d610a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:57:17.815 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:17.822+0000 7f48d610a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:57:18.202 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:17.836+0000 7f06b733f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T22:57:18.202 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:17.982+0000 7f06b733f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T22:57:18.202 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:18 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:18.085+0000 7f06b733f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T22:57:18.202 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:18 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:18.210+0000 7f06b733f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T22:57:18.605 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:18.128+0000 7f48d610a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:57:18.954 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:18.740+0000 7f48d610a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:57:18.983 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:18 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:18.527+0000 7f06b733f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T22:57:19.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:19 smithi138 ceph-mon[67823]: Standby manager daemon x restarted 2024-05-20T22:57:19.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:19 smithi138 ceph-mon[67823]: Standby manager daemon x started 2024-05-20T22:57:19.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:19 smithi138 ceph-mon[67823]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:57:19.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:19 smithi138 ceph-mon[67823]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:57:19.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:19 smithi138 ceph-mon[67823]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:57:19.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:19 smithi138 ceph-mon[67823]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:57:19.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:18.962+0000 7f48d610a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:57:19.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: [20/May/2024:22:57:18] ENGINE Bus STARTING 2024-05-20T22:57:19.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: CherryPy Checker: 2024-05-20T22:57:19.356 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: The Application mounted at '' has an empty config. 2024-05-20T22:57:19.357 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: 2024-05-20T22:57:19.357 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: [20/May/2024:22:57:19] ENGINE Serving on http://:::9283 2024-05-20T22:57:19.357 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 22:57:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x[60795]: [20/May/2024:22:57:19] ENGINE Bus STARTED 2024-05-20T22:57:19.410 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:19.182+0000 7f06b733f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T22:57:19.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[29498]: Standby manager daemon x restarted 2024-05-20T22:57:19.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[29498]: Standby manager daemon x started 2024-05-20T22:57:19.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:57:19.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:57:19.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:57:19.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[29498]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:57:19.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[86116]: Standby manager daemon x restarted 2024-05-20T22:57:19.412 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[86116]: Standby manager daemon x started 2024-05-20T22:57:19.412 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[86116]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-20T22:57:19.412 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[86116]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:57:19.412 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[86116]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-20T22:57:19.412 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:19 smithi055 ceph-mon[86116]: from='mgr.? 172.21.15.138:0/507108536' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:57:19.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:19.419+0000 7f06b733f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T22:57:19.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:19] ENGINE Bus STARTING 2024-05-20T22:57:19.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CherryPy Checker: 2024-05-20T22:57:19.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: The Application mounted at '' has an empty config. 2024-05-20T22:57:19.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:19.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:19] ENGINE Serving on http://:::9283 2024-05-20T22:57:19.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:19] ENGINE Bus STARTED 2024-05-20T22:57:20.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: mgrmap e45: y(active, since 4m), standbys: x 2024-05-20T22:57:20.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: Active manager daemon y restarted 2024-05-20T22:57:20.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: Activating manager daemon y 2024-05-20T22:57:20.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: osdmap e114: 8 total, 8 up, 8 in 2024-05-20T22:57:20.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: mgrmap e46: y(active, starting, since 0.0088257s), standbys: x 2024-05-20T22:57:20.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:57:20.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:57:20.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:57:20.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:57:20.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: Manager daemon y is now available 2024-05-20T22:57:20.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:20.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:57:20.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:57:20.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:57:20.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:57:20.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:20 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:57:20.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:20 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:20.434+0000 7f06747a2700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-05-20T22:57:20.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: mgrmap e45: y(active, since 4m), standbys: x 2024-05-20T22:57:20.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: Active manager daemon y restarted 2024-05-20T22:57:20.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: Activating manager daemon y 2024-05-20T22:57:20.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: osdmap e114: 8 total, 8 up, 8 in 2024-05-20T22:57:20.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: mgrmap e46: y(active, starting, since 0.0088257s), standbys: x 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:57:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: Manager daemon y is now available 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:57:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: mgrmap e45: y(active, since 4m), standbys: x 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: Active manager daemon y restarted 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: Activating manager daemon y 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: osdmap e114: 8 total, 8 up, 8 in 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: mgrmap e46: y(active, starting, since 0.0088257s), standbys: x 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T22:57:20.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: Manager daemon y is now available 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:20.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:57:20.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:57:20.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:57:20.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:57:20.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:20 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-20T22:57:21.287 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:21 smithi138 ceph-mon[67823]: [20/May/2024:22:57:19] ENGINE Bus STARTING 2024-05-20T22:57:21.287 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:21 smithi138 ceph-mon[67823]: [20/May/2024:22:57:20] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:57:21.287 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:21 smithi138 ceph-mon[67823]: [20/May/2024:22:57:20] ENGINE Serving on http://172.21.15.55:8765 2024-05-20T22:57:21.288 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:21 smithi138 ceph-mon[67823]: [20/May/2024:22:57:20] ENGINE Bus STARTED 2024-05-20T22:57:21.288 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:21 smithi138 ceph-mon[67823]: mgrmap e47: y(active, since 1.01207s), standbys: x 2024-05-20T22:57:21.310 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[86116]: [20/May/2024:22:57:19] ENGINE Bus STARTING 2024-05-20T22:57:21.311 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[86116]: [20/May/2024:22:57:20] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:57:21.311 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[86116]: [20/May/2024:22:57:20] ENGINE Serving on http://172.21.15.55:8765 2024-05-20T22:57:21.311 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[86116]: [20/May/2024:22:57:20] ENGINE Bus STARTED 2024-05-20T22:57:21.311 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[86116]: mgrmap e47: y(active, since 1.01207s), standbys: x 2024-05-20T22:57:21.312 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[29498]: [20/May/2024:22:57:19] ENGINE Bus STARTING 2024-05-20T22:57:21.312 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[29498]: [20/May/2024:22:57:20] ENGINE Serving on https://172.21.15.55:7150 2024-05-20T22:57:21.312 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[29498]: [20/May/2024:22:57:20] ENGINE Serving on http://172.21.15.55:8765 2024-05-20T22:57:21.312 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[29498]: [20/May/2024:22:57:20] ENGINE Bus STARTED 2024-05-20T22:57:21.312 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:21 smithi055 ceph-mon[29498]: mgrmap e47: y(active, since 1.01207s), standbys: x 2024-05-20T22:57:22.286 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:22 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:22.287 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:22 smithi055 ceph-mon[29498]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:57:22.287 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:22 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:22.287 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:22 smithi055 ceph-mon[86116]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:57:22.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:22 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:22.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:22 smithi138 ceph-mon[67823]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:57:23.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:23 smithi138 ceph-mon[67823]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:57:23.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:23 smithi138 ceph-mon[67823]: mgrmap e48: y(active, since 2s), standbys: x 2024-05-20T22:57:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:23 smithi055 ceph-mon[86116]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:57:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:23 smithi055 ceph-mon[86116]: mgrmap e48: y(active, since 2s), standbys: x 2024-05-20T22:57:23.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:23 smithi055 ceph-mon[29498]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:57:23.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:23 smithi055 ceph-mon[29498]: mgrmap e48: y(active, since 2s), standbys: x 2024-05-20T22:57:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[86116]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:57:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[86116]: mgrmap e49: y(active, since 4s), standbys: x 2024-05-20T22:57:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[29498]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:57:25.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[29498]: mgrmap e49: y(active, since 4s), standbys: x 2024-05-20T22:57:25.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:24 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:24 smithi138 ceph-mon[67823]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:57:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:24 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:24 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:24 smithi138 ceph-mon[67823]: mgrmap e49: y(active, since 4s), standbys: x 2024-05-20T22:57:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:24 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:24 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:26 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:26 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:26 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:26 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:26 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:26 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:26 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:26 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:26 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:26 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:26 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:26.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:26 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:27.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:27 smithi138 ceph-mon[67823]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T22:57:27.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:27 smithi055 ceph-mon[86116]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T22:57:27.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:27 smithi055 ceph-mon[29498]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T22:57:27.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:57:27] "GET /metrics HTTP/1.1" 200 35132 "" "Prometheus/2.43.0" 2024-05-20T22:57:29.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[29498]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:57:29.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:29.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:29.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:29.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:57:29.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[86116]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:57:29.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:29.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:29.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:29.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:29 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:57:29.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:29 smithi138 ceph-mon[67823]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-20T22:57:29.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:29 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:29.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:29 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:29.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:29 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:29.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:29 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:57:30.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:57:30.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:57:30.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:57:30.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:57:30.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:57:30.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:57:30.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.440 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.440 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:30.440 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:30.440 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:57:30.440 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-20T22:57:30.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:57:30.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:57:30.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:30 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-20T22:57:30.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: Updating smithi055:/etc/ceph/ceph.conf 2024-05-20T22:57:30.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: Updating smithi138:/etc/ceph/ceph.conf 2024-05-20T22:57:30.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:57:30.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:57:30.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:57:30.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:57:30.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:30.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:30.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:30.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:57:30.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:30 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-20T22:57:31.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:31 smithi055 ceph-mon[86116]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:57:31.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:31 smithi055 ceph-mon[86116]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:57:31.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:31 smithi055 ceph-mon[86116]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:57:31.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:31 smithi055 ceph-mon[86116]: Upgrade: It appears safe to stop mon.c 2024-05-20T22:57:31.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:31 smithi055 ceph-mon[29498]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:57:31.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:31 smithi055 ceph-mon[29498]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:57:31.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:31 smithi055 ceph-mon[29498]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:57:31.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:31 smithi055 ceph-mon[29498]: Upgrade: It appears safe to stop mon.c 2024-05-20T22:57:31.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:31 smithi138 ceph-mon[67823]: Updating smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:57:31.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:31 smithi138 ceph-mon[67823]: Updating smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:57:31.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:31 smithi138 ceph-mon[67823]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-20T22:57:31.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:31 smithi138 ceph-mon[67823]: Upgrade: It appears safe to stop mon.c 2024-05-20T22:57:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[86116]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:32.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[29498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:32.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[29498]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:32.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:32.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:32.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:32 smithi055 ceph-mon[29498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:32.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:32 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:32.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:32 smithi138 ceph-mon[67823]: from='mgr.54108 ' entity='mgr.y' 2024-05-20T22:57:32.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:32 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:32.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:32 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:32.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:32 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:33.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:33 smithi055 ceph-mon[86116]: Upgrade: Updating mon.c 2024-05-20T22:57:33.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:33 smithi055 ceph-mon[86116]: Deploying daemon mon.c on smithi055 2024-05-20T22:57:33.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:33 smithi055 ceph-mon[86116]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:57:33.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:33 smithi055 ceph-mon[29498]: Upgrade: Updating mon.c 2024-05-20T22:57:33.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:33 smithi055 ceph-mon[29498]: Deploying daemon mon.c on smithi055 2024-05-20T22:57:33.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:33 smithi055 ceph-mon[29498]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:57:33.532 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:33 smithi055 systemd[1]: Stopping Ceph mon.c for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:57:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:33 smithi138 ceph-mon[67823]: Upgrade: Updating mon.c 2024-05-20T22:57:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:33 smithi138 ceph-mon[67823]: Deploying daemon mon.c on smithi055 2024-05-20T22:57:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:33 smithi138 ceph-mon[67823]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-20T22:57:33.918 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:33] ENGINE Bus STOPPING 2024-05-20T22:57:33.918 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T22:57:33.918 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:33] ENGINE Bus STOPPED 2024-05-20T22:57:33.918 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:33] ENGINE Bus STARTING 2024-05-20T22:57:33.918 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:33] ENGINE Serving on http://:::9283 2024-05-20T22:57:33.918 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:22:57:33] ENGINE Bus STARTED 2024-05-20T22:57:33.919 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c[29494]: 2024-05-20T22:57:33.538+0000 7f106eb2e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:57:33.919 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c[29494]: 2024-05-20T22:57:33.538+0000 7f106eb2e700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-05-20T22:57:33.919 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:33 smithi055 podman[88326]: 2024-05-20 22:57:33.632673277 +0000 UTC m=+0.219165480 container died 79201364d4c8617d9071063bdbc3c8275061aaac18f1b7e706c2a3a3ba68ab2d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, version=8, maintainer=Guillaume Abrioux , ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, architecture=x86_64, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-20T22:57:34.209 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:34 smithi055 podman[88326]: 2024-05-20 22:57:34.008790357 +0000 UTC m=+0.595282563 container cleanup 79201364d4c8617d9071063bdbc3c8275061aaac18f1b7e706c2a3a3ba68ab2d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, GIT_BRANCH=HEAD, ceph=True, version=8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vcs-type=git, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc.) 2024-05-20T22:57:34.210 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:34 smithi055 bash[88326]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c 2024-05-20T22:57:34.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:34 smithi055 podman[88352]: 2024-05-20 22:57:34.218343275 +0000 UTC m=+0.584926347 container remove 79201364d4c8617d9071063bdbc3c8275061aaac18f1b7e706c2a3a3ba68ab2d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, ceph=True, io.openshift.expose-services=, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, vcs-type=git, name=centos-stream, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, architecture=x86_64, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-20T22:57:34.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:34 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.c.service: Deactivated successfully. 2024-05-20T22:57:34.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:34 smithi055 systemd[1]: Stopped Ceph mon.c for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:57:34.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:34 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.c.service: Consumed 16.834s CPU time. 2024-05-20T22:57:34.965 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:34 smithi055 systemd[1]: Starting Ceph mon.c for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:57:34.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:34 smithi055 podman[88480]: 2024-05-20 22:57:34.710391485 +0000 UTC m=+0.018501377 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:57:34.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:34 smithi055 podman[88480]: 2024-05-20 22:57:34.828236763 +0000 UTC m=+0.136346634 container create afab2b40c96b24dfed361f4aa4652c123e61274d8eab14e01d8e0aa248ea7af4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-05-20T22:57:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 podman[88480]: 2024-05-20 22:57:35.072591633 +0000 UTC m=+0.380701513 container init afab2b40c96b24dfed361f4aa4652c123e61274d8eab14e01d8e0aa248ea7af4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-05-20T22:57:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 podman[88480]: 2024-05-20 22:57:35.076282479 +0000 UTC m=+0.384392365 container start afab2b40c96b24dfed361f4aa4652c123e61274d8eab14e01d8e0aa248ea7af4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-05-20T22:57:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T22:57:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev), process ceph-mon, pid 2 2024-05-20T22:57:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: pidfile_write: ignore empty --pid-file 2024-05-20T22:57:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: load: jerasure load: lrc 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: RocksDB version: 7.9.2 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Git sha 0 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Compile date 2024-05-17 12:39:40 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: DB SUMMARY 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: DB Session ID: VIJR39CIRSYLHO3MKVRW 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: CURRENT file: CURRENT 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3209 Bytes 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2592221 ; 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.error_if_exists: 0 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.create_if_missing: 0 2024-05-20T22:57:35.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.paranoid_checks: 1 2024-05-20T22:57:35.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-20T22:57:35.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T22:57:35.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-20T22:57:35.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.env: 0x5626deaa9c60 2024-05-20T22:57:35.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.fs: PosixFileSystem 2024-05-20T22:57:35.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.info_log: 0x5626df3b5ac0 2024-05-20T22:57:35.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T22:57:35.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.statistics: (nil) 2024-05-20T22:57:35.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.use_fsync: 0 2024-05-20T22:57:35.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_log_file_size: 0 2024-05-20T22:57:35.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.allow_fallocate: 1 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.use_direct_reads: 0 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.db_log_dir: 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.wal_dir: 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T22:57:35.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.write_buffer_manager: 0x5626df388820 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T22:57:35.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.unordered_write: 0 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.row_cache: None 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.wal_filter: None 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.two_write_queues: 0 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T22:57:35.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.wal_compression: 0 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.atomic_flush: 0 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.log_readahead_size: 0 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_background_jobs: 2 2024-05-20T22:57:35.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_background_compactions: -1 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_subcompactions: 1 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_open_files: -1 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T22:57:35.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_background_flushes: -1 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Compression algorithms supported: 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: kZSTD supported: 0 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: kXpressCompression supported: 0 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: kZlibCompression supported: 1 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: kSnappyCompression supported: 1 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: kLZ4Compression supported: 1 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: kBZip2Compression supported: 0 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-20T22:57:35.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.merge_operator: 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_filter: None 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5626df3b5bc0) 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: cache_index_and_filter_blocks: 1 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: pin_top_level_index_and_filter: 1 2024-05-20T22:57:35.243 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_type: 0 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: data_block_index_type: 0 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_shortening: 1 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: checksum: 4 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: no_block_cache: 0 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache: 0x5626df3b9090 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_name: BinnedLRUCache 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_options: 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: capacity : 536870912 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: num_shard_bits : 4 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: strict_capacity_limit : 0 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: high_pri_pool_ratio: 0.000 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_compressed: (nil) 2024-05-20T22:57:35.244 INFO:journalctl@ceph.mon.c.smithi055.stdout: persistent_cache: (nil) 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_size: 4096 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_size_deviation: 10 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_restart_interval: 16 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_block_restart_interval: 1 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: metadata_block_size: 4096 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: partition_filters: 0 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_delta_encoding: 1 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: filter_policy: bloomfilter 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: whole_key_filtering: 1 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: verify_compression: 0 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: read_amp_bytes_per_bit: 0 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: format_version: 5 2024-05-20T22:57:35.245 INFO:journalctl@ceph.mon.c.smithi055.stdout: enable_index_compression: 1 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_align: 0 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout: max_auto_readahead_size: 262144 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout: prepopulate_block_cache: 0 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout: initial_auto_readahead_size: 8192 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout: num_file_reads_for_auto_readahead: 2 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression: NoCompression 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.num_levels: 7 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T22:57:35.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T22:57:35.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T22:57:35.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T22:57:35.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T22:57:35.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T22:57:35.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T22:57:35.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T22:57:35.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T22:57:35.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T22:57:35.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T22:57:35.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.arena_block_size: 1048576 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T22:57:35.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.table_properties_collectors: 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.inplace_update_support: 0 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.bloom_locality: 0 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.max_successive_merges: 0 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T22:57:35.251 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.ttl: 2592000 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.enable_blob_files: false 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.min_blob_size: 0 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-20T22:57:35.252 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.blob_file_starting_level: 0 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32797, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 45cee8c3-08c7-45fc-bc80-49d57f8f1da6 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716245855106192, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716245855113121, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1659386, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32803, "largest_seqno": 33219, "table_properties": {"data_size": 1656506, "index_size": 1354, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4733, "raw_average_key_size": 24, "raw_value_size": 1652047, "raw_average_value_size": 8386, "num_data_blocks": 61, "num_entries": 197, "num_filter_entries": 197, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716245855, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "45cee8c3-08c7-45fc-bc80-49d57f8f1da6", "db_session_id": "VIJR39CIRSYLHO3MKVRW", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716245855113219, "job": 1, "event": "recovery_finished"} 2024-05-20T22:57:35.253 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5626df4aa000 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: DB pointer 0x5626df494000 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** DB Stats ** 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:57:35.254 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: L0 1/0 1.58 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 535.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: L6 1/0 10.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: Sum 2/0 11.97 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 535.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 535.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2024-05-20T22:57:35.255 INFO:journalctl@ceph.mon.c.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 535.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative compaction: 0.00 GB write, 161.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval compaction: 0.00 GB write, 161.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:57:35.256 INFO:journalctl@ceph.mon.c.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: Block cache BinnedLRUCache@0x5626df3b9090#2 capacity: 512.00 MB usage: 2.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: starting mon.c rank 1 at public addrs [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] at bind addrs [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: mon.c@-1(???) e3 preinit fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: mon.c@-1(???).mds e1 new map 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: mon.c@-1(???).mds e1 print_map 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: e1 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: legacy client fscid: -1 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2024-05-20T22:57:35.257 INFO:journalctl@ceph.mon.c.smithi055.stdout: No filesystems configured 2024-05-20T22:57:35.258 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: mon.c@-1(???).osd e114 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-20T22:57:35.258 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: mon.c@-1(???).osd e114 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:57:35.258 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: mon.c@-1(???).osd e114 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:57:35.258 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: mon.c@-1(???).osd e114 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:57:35.258 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 ceph-mon[88498]: mon.c@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-05-20T22:57:35.258 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 bash[88480]: afab2b40c96b24dfed361f4aa4652c123e61274d8eab14e01d8e0aa248ea7af4 2024-05-20T22:57:35.258 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:35 smithi055 systemd[1]: Started Ceph mon.c for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:57:36.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:57:36.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: 0: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.a 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: 1: [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon.c 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: 2: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.b 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: fsmap 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: osdmap e114: 8 total, 8 up, 8 in 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: mgrmap e49: y(active, since 16s), standbys: x 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: overall HEALTH_OK 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: mon.c calling monitor election 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: mon.b calling monitor election 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: mon.a calling monitor election 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:57:36.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: monmap epoch 4 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: last_changed 2024-05-20T22:57:35.544643+0000 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: created 2024-05-20T22:26:07.295242+0000 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: min_mon_release 19 (squid) 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: election_strategy: 1 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: 0: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.a 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: 1: [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon.c 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: 2: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.b 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: fsmap 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: osdmap e114: 8 total, 8 up, 8 in 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: mgrmap e49: y(active, since 16s), standbys: x 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: overall HEALTH_OK 2024-05-20T22:57:36.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: 0: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.a 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: 1: [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon.c 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: 2: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.b 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: fsmap 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: osdmap e114: 8 total, 8 up, 8 in 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: mgrmap e49: y(active, since 16s), standbys: x 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: overall HEALTH_OK 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: mon.c calling monitor election 2024-05-20T22:57:36.653 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: mon.b calling monitor election 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: mon.a calling monitor election 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: monmap epoch 4 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: last_changed 2024-05-20T22:57:35.544643+0000 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: created 2024-05-20T22:26:07.295242+0000 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: min_mon_release 19 (squid) 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: election_strategy: 1 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: 0: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.a 2024-05-20T22:57:36.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: 1: [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon.c 2024-05-20T22:57:36.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: 2: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.b 2024-05-20T22:57:36.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: fsmap 2024-05-20T22:57:36.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: osdmap e114: 8 total, 8 up, 8 in 2024-05-20T22:57:36.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: mgrmap e49: y(active, since 16s), standbys: x 2024-05-20T22:57:36.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: overall HEALTH_OK 2024-05-20T22:57:36.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:36.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:36.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:36 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: 0: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.a 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: 1: [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon.c 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: 2: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.b 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: fsmap 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: osdmap e114: 8 total, 8 up, 8 in 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: mgrmap e49: y(active, since 16s), standbys: x 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: overall HEALTH_OK 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: mon.c calling monitor election 2024-05-20T22:57:36.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: mon.b calling monitor election 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: mon.a calling monitor election 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: monmap epoch 4 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: last_changed 2024-05-20T22:57:35.544643+0000 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: created 2024-05-20T22:26:07.295242+0000 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: min_mon_release 19 (squid) 2024-05-20T22:57:36.857 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: election_strategy: 1 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: 0: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.a 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: 1: [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon.c 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: 2: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.b 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: fsmap 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: osdmap e114: 8 total, 8 up, 8 in 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: mgrmap e49: y(active, since 16s), standbys: x 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: overall HEALTH_OK 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:36.858 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:36 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:57:37.002 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:57:37.278 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:57:37] "GET /metrics HTTP/1.1" 200 37935 "" "Prometheus/2.43.0" 2024-05-20T22:57:37.705 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:57:37.705 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (2m) 13s ago 24m 26.7M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T22:57:37.706 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (2m) 13s ago 24m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T22:57:37.706 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (2m) 13s ago 24m 47.5M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T22:57:37.706 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (4m) 13s ago 29m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:57:37.706 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (14m) 13s ago 31m 476M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:57:37.706 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (27s) 13s ago 31m 30.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T22:57:37.706 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (77s) 13s ago 29m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T22:57:37.707 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 starting - - - 2048M 2024-05-20T22:57:37.707 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (3m) 13s ago 25m 15.8M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:57:37.707 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (3m) 13s ago 25m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:57:37.707 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (29m) 13s ago 29m 55.6M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:57:37.707 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (28m) 13s ago 28m 55.5M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:57:37.707 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (28m) 13s ago 28m 53.8M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:57:37.707 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (28m) 13s ago 28m 53.5M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:57:37.708 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (27m) 13s ago 27m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:57:37.708 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (27m) 13s ago 27m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:57:37.708 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (26m) 13s ago 26m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:57:37.708 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (26m) 13s ago 26m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:57:37.708 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (3m) 13s ago 25m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:57:37.708 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (24m) 13s ago 24m 76.3M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:57:37.708 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (24m) 13s ago 24m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:57:37.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:37 smithi138 ceph-mon[67823]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:37.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:37 smithi055 ceph-mon[86116]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:37.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:37 smithi055 ceph-mon[88498]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:38.141 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:57:38.141 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:57:38.142 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-20T22:57:38.143 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 5 2024-05-20T22:57:38.143 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:57:38.143 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:57:38.493 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:57:38.493 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:57:38.494 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:57:38.494 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi055", 2024-05-20T22:57:38.494 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T22:57:38.494 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "1/2 daemons upgraded", 2024-05-20T22:57:38.494 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading mon daemons", 2024-05-20T22:57:38.494 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:57:38.494 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:57:38.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:38 smithi055 ceph-mon[88498]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:38.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:38 smithi055 ceph-mon[88498]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:57:38.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:38 smithi055 ceph-mon[88498]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:38.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:38 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2665896490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:38.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:38 smithi055 ceph-mon[86116]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:38.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:38 smithi055 ceph-mon[86116]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:57:38.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:38 smithi055 ceph-mon[86116]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:38.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:38 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2665896490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:38.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:38 smithi138 ceph-mon[67823]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:38.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:38 smithi138 ceph-mon[67823]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-20T22:57:38.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:38 smithi138 ceph-mon[67823]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:38.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:38 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2665896490' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:57:39.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:39 smithi138 ceph-mon[67823]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:39.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:39 smithi055 ceph-mon[86116]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:39.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:39 smithi055 ceph-mon[88498]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:57:40.449 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:57:40.121+0000 7f06747a2700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-05-20T22:57:41.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:40 smithi055 ceph-mon[86116]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:41.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:40 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:41.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:40 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:41.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:40 smithi055 ceph-mon[88498]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:41.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:40 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:41.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:40 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:41.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:40 smithi138 ceph-mon[67823]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:41.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:40 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:41.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:40 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:42.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:42 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:42.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:42 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:42.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:42 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:42.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:42 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:42.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:42 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:42.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:42 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:42.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:42 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:42.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:42 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:42.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:42 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:43.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:43 smithi055 ceph-mon[86116]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:43.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:43 smithi055 ceph-mon[88498]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:43.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:43 smithi138 ceph-mon[67823]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:45.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[86116]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:45.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:45.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:45.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:45.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:57:45.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:45.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[88498]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:45.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:45.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:45 smithi138 ceph-mon[67823]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:45.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:45.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:45.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:45.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:57:45.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:45.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:45.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:45.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:46.330 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:46 smithi055 ceph-mon[86116]: Detected new or changed devices on smithi055 2024-05-20T22:57:46.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:46 smithi055 ceph-mon[86116]: Reconfiguring mon.a (monmap changed)... 2024-05-20T22:57:46.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:46 smithi055 ceph-mon[86116]: Reconfiguring daemon mon.a on smithi055 2024-05-20T22:57:46.331 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:46 smithi055 ceph-mon[88498]: Detected new or changed devices on smithi055 2024-05-20T22:57:46.331 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:46 smithi055 ceph-mon[88498]: Reconfiguring mon.a (monmap changed)... 2024-05-20T22:57:46.331 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:46 smithi055 ceph-mon[88498]: Reconfiguring daemon mon.a on smithi055 2024-05-20T22:57:46.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:46 smithi138 ceph-mon[67823]: Detected new or changed devices on smithi055 2024-05-20T22:57:46.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:46 smithi138 ceph-mon[67823]: Reconfiguring mon.a (monmap changed)... 2024-05-20T22:57:46.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:46 smithi138 ceph-mon[67823]: Reconfiguring daemon mon.a on smithi055 2024-05-20T22:57:47.343 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:57:47] "GET /metrics HTTP/1.1" 200 37935 "" "Prometheus/2.43.0" 2024-05-20T22:57:47.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[86116]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:47.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:47.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:47.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:57:47.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:57:47.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:47.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[88498]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:47.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:47.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:47.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:57:47.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:57:47.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:47 smithi138 ceph-mon[67823]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:57:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:57:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:48.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:48 smithi055 ceph-mon[86116]: Reconfiguring mgr.y (monmap changed)... 2024-05-20T22:57:48.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:48 smithi055 ceph-mon[86116]: Reconfiguring daemon mgr.y on smithi055 2024-05-20T22:57:48.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:48 smithi055 ceph-mon[88498]: Reconfiguring mgr.y (monmap changed)... 2024-05-20T22:57:48.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:48 smithi055 ceph-mon[88498]: Reconfiguring daemon mgr.y on smithi055 2024-05-20T22:57:48.506 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:48 smithi138 ceph-mon[67823]: Reconfiguring mgr.y (monmap changed)... 2024-05-20T22:57:48.507 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:48 smithi138 ceph-mon[67823]: Reconfiguring daemon mgr.y on smithi055 2024-05-20T22:57:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[86116]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:49.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[88498]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:49.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:49.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:49.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:49.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:49.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:49.519 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:49 smithi138 ceph-mon[67823]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:49.519 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:49.519 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:49.520 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:49.520 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:49.520 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:50.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[88498]: Reconfiguring mon.c (monmap changed)... 2024-05-20T22:57:50.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[88498]: Reconfiguring daemon mon.c on smithi055 2024-05-20T22:57:50.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:57:50.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:50.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:50.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:57:50.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:50.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[86116]: Reconfiguring mon.c (monmap changed)... 2024-05-20T22:57:50.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[86116]: Reconfiguring daemon mon.c on smithi055 2024-05-20T22:57:50.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:57:50.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:50.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:50.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:57:50.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:50.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:50 smithi138 ceph-mon[67823]: Reconfiguring mon.c (monmap changed)... 2024-05-20T22:57:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:50 smithi138 ceph-mon[67823]: Reconfiguring daemon mon.c on smithi055 2024-05-20T22:57:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:57:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:57:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:51.458 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:51 smithi055 ceph-mon[86116]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:51.459 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:51 smithi055 ceph-mon[86116]: Reconfiguring osd.0 (monmap changed)... 2024-05-20T22:57:51.459 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:51 smithi055 ceph-mon[86116]: Reconfiguring daemon osd.0 on smithi055 2024-05-20T22:57:51.459 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:51 smithi055 ceph-mon[88498]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:51.459 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:51 smithi055 ceph-mon[88498]: Reconfiguring osd.0 (monmap changed)... 2024-05-20T22:57:51.459 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:51 smithi055 ceph-mon[88498]: Reconfiguring daemon osd.0 on smithi055 2024-05-20T22:57:51.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:51 smithi138 ceph-mon[67823]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:51.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:51 smithi138 ceph-mon[67823]: Reconfiguring osd.0 (monmap changed)... 2024-05-20T22:57:51.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:51 smithi138 ceph-mon[67823]: Reconfiguring daemon osd.0 on smithi055 2024-05-20T22:57:52.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:52.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:52.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:52.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:57:52.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:52.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:52.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:52.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:52.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:57:52.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:52.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:52 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:57:52.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:52.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:52.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:57:52.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:53.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:53 smithi055 ceph-mon[86116]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:53.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:53 smithi055 ceph-mon[86116]: Reconfiguring osd.1 (monmap changed)... 2024-05-20T22:57:53.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:53 smithi055 ceph-mon[86116]: Reconfiguring daemon osd.1 on smithi055 2024-05-20T22:57:53.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:53 smithi055 ceph-mon[88498]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:53.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:53 smithi055 ceph-mon[88498]: Reconfiguring osd.1 (monmap changed)... 2024-05-20T22:57:53.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:53 smithi055 ceph-mon[88498]: Reconfiguring daemon osd.1 on smithi055 2024-05-20T22:57:53.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:53 smithi138 ceph-mon[67823]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:53.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:53 smithi138 ceph-mon[67823]: Reconfiguring osd.1 (monmap changed)... 2024-05-20T22:57:53.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:53 smithi138 ceph-mon[67823]: Reconfiguring daemon osd.1 on smithi055 2024-05-20T22:57:54.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[86116]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:54.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:54.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:54.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[86116]: Reconfiguring osd.2 (monmap changed)... 2024-05-20T22:57:54.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:57:54.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:54.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[86116]: Reconfiguring daemon osd.2 on smithi055 2024-05-20T22:57:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[88498]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[88498]: Reconfiguring osd.2 (monmap changed)... 2024-05-20T22:57:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:57:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:54.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:54 smithi055 ceph-mon[88498]: Reconfiguring daemon osd.2 on smithi055 2024-05-20T22:57:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:54 smithi138 ceph-mon[67823]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:54 smithi138 ceph-mon[67823]: Reconfiguring osd.2 (monmap changed)... 2024-05-20T22:57:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:57:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:54.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:54 smithi138 ceph-mon[67823]: Reconfiguring daemon osd.2 on smithi055 2024-05-20T22:57:56.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:56.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:56.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[88498]: Reconfiguring osd.3 (monmap changed)... 2024-05-20T22:57:56.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:57:56.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:56.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[88498]: Reconfiguring daemon osd.3 on smithi055 2024-05-20T22:57:56.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[88498]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:56.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:56.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:56.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[86116]: Reconfiguring osd.3 (monmap changed)... 2024-05-20T22:57:56.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:57:56.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:56.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[86116]: Reconfiguring daemon osd.3 on smithi055 2024-05-20T22:57:56.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:56 smithi055 ceph-mon[86116]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:56.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:56 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:56.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:56 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:56.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:56 smithi138 ceph-mon[67823]: Reconfiguring osd.3 (monmap changed)... 2024-05-20T22:57:56.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:56 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:57:56.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:56 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:56.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:56 smithi138 ceph-mon[67823]: Reconfiguring daemon osd.3 on smithi055 2024-05-20T22:57:56.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:56 smithi138 ceph-mon[67823]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:57:57.635 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:57:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:57:57] "GET /metrics HTTP/1.1" 200 37947 "" "Prometheus/2.43.0" 2024-05-20T22:57:58.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:58.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:58.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[88498]: Reconfiguring rgw.foo.smithi055.ymnqng (monmap changed)... 2024-05-20T22:57:58.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:57:58.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:58.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[88498]: Reconfiguring daemon rgw.foo.smithi055.ymnqng on smithi055 2024-05-20T22:57:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[86116]: Reconfiguring rgw.foo.smithi055.ymnqng (monmap changed)... 2024-05-20T22:57:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:57:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:58 smithi055 ceph-mon[86116]: Reconfiguring daemon rgw.foo.smithi055.ymnqng on smithi055 2024-05-20T22:57:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:58 smithi138 ceph-mon[67823]: Reconfiguring rgw.foo.smithi055.ymnqng (monmap changed)... 2024-05-20T22:57:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:57:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:58 smithi138 ceph-mon[67823]: Reconfiguring daemon rgw.foo.smithi055.ymnqng on smithi055 2024-05-20T22:57:59.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[88498]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:59.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:59.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:59.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:59.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:59.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[86116]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:57:59 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:57:59.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:59 smithi138 ceph-mon[67823]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:57:59.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:59 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:59.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:59 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:57:59.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:59 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:57:59.340 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:59 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:57:59.341 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:57:59 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:00.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:00 smithi055 ceph-mon[86116]: Reconfiguring mon.b (monmap changed)... 2024-05-20T22:58:00.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:00 smithi055 ceph-mon[86116]: Reconfiguring daemon mon.b on smithi138 2024-05-20T22:58:00.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:00 smithi055 ceph-mon[88498]: Reconfiguring mon.b (monmap changed)... 2024-05-20T22:58:00.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:00 smithi055 ceph-mon[88498]: Reconfiguring daemon mon.b on smithi138 2024-05-20T22:58:00.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:00 smithi138 ceph-mon[67823]: Reconfiguring mon.b (monmap changed)... 2024-05-20T22:58:00.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:00 smithi138 ceph-mon[67823]: Reconfiguring daemon mon.b on smithi138 2024-05-20T22:58:01.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:01 smithi138 ceph-mon[67823]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:01.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:01 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:01.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:01 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:01.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:01 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:58:01.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:01 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:58:01.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:01 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:01.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[86116]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:01.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:01.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:01.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:58:01.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:58:01.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:01.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[88498]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:01.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:01.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:01.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:58:01.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:58:01.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:01 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:02.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:02 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:02.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:02 smithi138 ceph-mon[67823]: Reconfiguring mgr.x (monmap changed)... 2024-05-20T22:58:02.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:02 smithi138 ceph-mon[67823]: Reconfiguring daemon mgr.x on smithi138 2024-05-20T22:58:02.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:02 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:02.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:02 smithi055 ceph-mon[86116]: Reconfiguring mgr.x (monmap changed)... 2024-05-20T22:58:02.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:02 smithi055 ceph-mon[86116]: Reconfiguring daemon mgr.x on smithi138 2024-05-20T22:58:02.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:02 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:02.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:02 smithi055 ceph-mon[88498]: Reconfiguring mgr.x (monmap changed)... 2024-05-20T22:58:02.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:02 smithi055 ceph-mon[88498]: Reconfiguring daemon mgr.x on smithi138 2024-05-20T22:58:03.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:03 smithi138 ceph-mon[67823]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:03.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:03 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:03 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:03 smithi138 ceph-mon[67823]: Reconfiguring osd.4 (monmap changed)... 2024-05-20T22:58:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:03 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:58:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:03 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:03.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:03 smithi138 ceph-mon[67823]: Reconfiguring daemon osd.4 on smithi138 2024-05-20T22:58:03.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[86116]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:03.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:03.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:03.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[86116]: Reconfiguring osd.4 (monmap changed)... 2024-05-20T22:58:03.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:58:03.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:03.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[86116]: Reconfiguring daemon osd.4 on smithi138 2024-05-20T22:58:03.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[88498]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:03.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:03.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:03.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[88498]: Reconfiguring osd.4 (monmap changed)... 2024-05-20T22:58:03.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:58:03.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:03.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:03 smithi055 ceph-mon[88498]: Reconfiguring daemon osd.4 on smithi138 2024-05-20T22:58:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:04 smithi138 ceph-mon[67823]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:04 smithi138 ceph-mon[67823]: Reconfiguring osd.5 (monmap changed)... 2024-05-20T22:58:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:58:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:04 smithi138 ceph-mon[67823]: Reconfiguring daemon osd.5 on smithi138 2024-05-20T22:58:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[86116]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[86116]: Reconfiguring osd.5 (monmap changed)... 2024-05-20T22:58:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:58:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[86116]: Reconfiguring daemon osd.5 on smithi138 2024-05-20T22:58:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:05.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[88498]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:05.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:05.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:05.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[88498]: Reconfiguring osd.5 (monmap changed)... 2024-05-20T22:58:05.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:58:05.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:05.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[88498]: Reconfiguring daemon osd.5 on smithi138 2024-05-20T22:58:05.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:06.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:06 smithi138 ceph-mon[67823]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:06.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:06 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:06 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:06 smithi138 ceph-mon[67823]: Reconfiguring osd.6 (monmap changed)... 2024-05-20T22:58:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:06 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:58:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:06 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:06.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:06 smithi138 ceph-mon[67823]: Reconfiguring daemon osd.6 on smithi138 2024-05-20T22:58:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[86116]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[86116]: Reconfiguring osd.6 (monmap changed)... 2024-05-20T22:58:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:58:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:06.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[86116]: Reconfiguring daemon osd.6 on smithi138 2024-05-20T22:58:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[88498]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[88498]: Reconfiguring osd.6 (monmap changed)... 2024-05-20T22:58:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:58:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:06.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:06 smithi055 ceph-mon[88498]: Reconfiguring daemon osd.6 on smithi138 2024-05-20T22:58:07.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:58:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:58:07] "GET /metrics HTTP/1.1" 200 37946 "" "Prometheus/2.43.0" 2024-05-20T22:58:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[88498]: Reconfiguring osd.7 (monmap changed)... 2024-05-20T22:58:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:58:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[88498]: Reconfiguring daemon osd.7 on smithi138 2024-05-20T22:58:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[88498]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:08.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:08.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:08.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[86116]: Reconfiguring osd.7 (monmap changed)... 2024-05-20T22:58:08.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:58:08.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:08.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[86116]: Reconfiguring daemon osd.7 on smithi138 2024-05-20T22:58:08.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:08 smithi055 ceph-mon[86116]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:08.510 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:08.510 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:08.510 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:08 smithi138 ceph-mon[67823]: Reconfiguring osd.7 (monmap changed)... 2024-05-20T22:58:08.511 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:58:08.511 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:08.511 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:08 smithi138 ceph-mon[67823]: Reconfiguring daemon osd.7 on smithi138 2024-05-20T22:58:08.511 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:08 smithi138 ceph-mon[67823]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:08.871 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:58:09.558 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:58:09.558 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (3m) 29s ago 25m 26.7M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T22:58:09.558 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (2m) 45s ago 25m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T22:58:09.558 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (2m) 29s ago 24m 50.4M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T22:58:09.558 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (4m) 45s ago 30m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:58:09.558 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (14m) 29s ago 31m 496M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:58:09.559 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (59s) 29s ago 31m 38.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T22:58:09.559 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (109s) 45s ago 30m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T22:58:09.559 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (34s) 29s ago 30m 20.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T22:58:09.559 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (4m) 29s ago 26m 15.6M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:58:09.559 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (4m) 45s ago 26m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:58:09.559 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (29m) 29s ago 29m 54.7M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:58:09.559 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (29m) 29s ago 29m 55.9M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:58:09.559 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (29m) 29s ago 29m 54.0M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:58:09.560 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (28m) 29s ago 28m 53.6M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:58:09.560 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (28m) 45s ago 28m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:58:09.560 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (27m) 45s ago 27m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:58:09.560 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (27m) 45s ago 27m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:58:09.560 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (27m) 45s ago 27m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:58:09.560 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (3m) 45s ago 26m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:58:09.561 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (24m) 29s ago 24m 76.5M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:58:09.561 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (24m) 45s ago 24m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:58:09.970 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:58:09.970 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:58:09.971 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:58:09.972 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-20T22:58:09.972 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 5 2024-05-20T22:58:09.972 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:58:09.972 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:58:10.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:09 smithi138 ceph-mon[67823]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:10.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:09 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:10.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:09 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:10.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:09 smithi138 ceph-mon[67823]: Reconfiguring rgw.foo.smithi138.sfpuja (monmap changed)... 2024-05-20T22:58:10.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:09 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:58:10.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:09 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:10.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:09 smithi138 ceph-mon[67823]: Reconfiguring daemon rgw.foo.smithi138.sfpuja on smithi138 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[86116]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[86116]: Reconfiguring rgw.foo.smithi138.sfpuja (monmap changed)... 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[86116]: Reconfiguring daemon rgw.foo.smithi138.sfpuja on smithi138 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[88498]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:10.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:10.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[88498]: Reconfiguring rgw.foo.smithi138.sfpuja (monmap changed)... 2024-05-20T22:58:10.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:58:10.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:10.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:09 smithi055 ceph-mon[88498]: Reconfiguring daemon rgw.foo.smithi138.sfpuja on smithi138 2024-05-20T22:58:10.325 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:58:10.326 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:58:10.326 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:58:10.326 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi055", 2024-05-20T22:58:10.326 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [ 2024-05-20T22:58:10.326 INFO:teuthology.orchestra.run.smithi055.stdout: "mon" 2024-05-20T22:58:10.326 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T22:58:10.326 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "2/2 daemons upgraded", 2024-05-20T22:58:10.326 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading mon daemons", 2024-05-20T22:58:10.326 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:58:10.327 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:58:11.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='client.54174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/333825202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='client.54174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/333825202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:58:11.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:58:11.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:58:11.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:58:11.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:58:11.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:58:11.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:58:11.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:58:11.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:58:11.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:58:11.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:58:11.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:58:11.246 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:10 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='client.54174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/333825202' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:58:11.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:58:11.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:58:11.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:58:11.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:58:11.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:58:11.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:58:11.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:58:11.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:58:11.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:58:11.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:58:11.362 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:10 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:58:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mon 2024-05-20T22:58:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all crash 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mds 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all nfs 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all node-exporter 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all prometheus 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all alertmanager 2024-05-20T22:58:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all grafana 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all loki 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all promtail 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Finalizing container_image settings 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: Upgrade: Complete! 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:12.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mon 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all crash 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mds 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all nfs 2024-05-20T22:58:12.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all node-exporter 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all prometheus 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all alertmanager 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all grafana 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all loki 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all promtail 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Finalizing container_image settings 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: Upgrade: Complete! 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:12.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:58:12.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:58:12.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:12.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:58:12.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mon 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all crash 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mds 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nfs 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all node-exporter 2024-05-20T22:58:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all prometheus 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all alertmanager 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all grafana 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all loki 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all promtail 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Finalizing container_image settings 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: Upgrade: Complete! 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:12.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:58:12.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:58:12.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:58:12.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:13.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:12 smithi055 ceph-mon[86116]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:13.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:12 smithi055 ceph-mon[88498]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:13.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:12 smithi138 ceph-mon[67823]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:15.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:14 smithi055 ceph-mon[86116]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:15.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:14 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:15.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:14 smithi055 ceph-mon[88498]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:15.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:14 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:15.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:14 smithi138 ceph-mon[67823]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:15.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:14 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:58:17.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:16 smithi055 ceph-mon[86116]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:17.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:16 smithi055 ceph-mon[88498]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:17.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:16 smithi138 ceph-mon[67823]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:17.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:58:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:58:17] "GET /metrics HTTP/1.1" 200 37946 "" "Prometheus/2.43.0" 2024-05-20T22:58:19.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:18 smithi055 ceph-mon[86116]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:19.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:18 smithi055 ceph-mon[88498]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:19.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:18 smithi138 ceph-mon[67823]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:20.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:19 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:20.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:19 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:20.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:19 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:21.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:20 smithi055 ceph-mon[86116]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:21.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:20 smithi055 ceph-mon[88498]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:21.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:20 smithi138 ceph-mon[67823]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:22.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:21 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:22.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:21 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:22.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:21 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:23.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:22 smithi138 ceph-mon[67823]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:23.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:22 smithi055 ceph-mon[86116]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:23.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:23 smithi055 ceph-mon[88498]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:25.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:25 smithi138 ceph-mon[67823]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:25.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:25 smithi055 ceph-mon[86116]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:25.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:25 smithi055 ceph-mon[88498]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:27.285 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:27 smithi055 ceph-mon[86116]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:27.285 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:27 smithi055 ceph-mon[88498]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:27.285 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:58:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:58:27] "GET /metrics HTTP/1.1" 200 37873 "" "Prometheus/2.43.0" 2024-05-20T22:58:27.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:27 smithi138 ceph-mon[67823]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:29.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:29 smithi138 ceph-mon[67823]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:29.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:29 smithi055 ceph-mon[86116]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:29.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:29 smithi055 ceph-mon[88498]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:31.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:31 smithi138 ceph-mon[67823]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:31.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:31 smithi055 ceph-mon[88498]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:31.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:31 smithi055 ceph-mon[86116]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:32.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:32 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:32.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:32 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:32.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:32 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:33.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:33 smithi138 ceph-mon[67823]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:33.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:33 smithi055 ceph-mon[86116]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:33.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:33 smithi055 ceph-mon[88498]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:35.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:35 smithi138 ceph-mon[67823]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:35.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:35 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:35.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:35 smithi055 ceph-mon[86116]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:35.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:35 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:35.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:35 smithi055 ceph-mon[88498]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:35.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:35 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:37.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:37 smithi138 ceph-mon[67823]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:37.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:37 smithi055 ceph-mon[86116]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:37.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:37 smithi055 ceph-mon[88498]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:37.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:58:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:58:37] "GET /metrics HTTP/1.1" 200 37939 "" "Prometheus/2.43.0" 2024-05-20T22:58:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:39 smithi138 ceph-mon[67823]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:39.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:39 smithi055 ceph-mon[86116]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:39 smithi055 ceph-mon[88498]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:41.330 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:41 smithi055 ceph-mon[86116]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:41.330 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:41 smithi055 ceph-mon[88498]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:41.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:41 smithi138 ceph-mon[67823]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:41.357 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T22:58:42.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:42 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:42.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:42 smithi055 ceph-mon[86116]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:42.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:42 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:42.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:42 smithi055 ceph-mon[88498]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:42.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:42 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:42.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:42 smithi138 ceph-mon[67823]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:43.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:43 smithi138 ceph-mon[67823]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:43.474 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:58:43.474 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (3m) 63s ago 26m 26.7M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (3m) 79s ago 25m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (3m) 63s ago 25m 50.4M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (5m) 79s ago 30m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (15m) 63s ago 32m 496M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (93s) 63s ago 32m 38.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (2m) 79s ago 31m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (68s) 63s ago 31m 20.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (4m) 63s ago 26m 15.6M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (4m) 79s ago 26m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (30m) 63s ago 30m 54.7M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (30m) 63s ago 30m 55.9M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (29m) 63s ago 29m 54.0M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:58:43.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (29m) 63s ago 29m 53.6M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:58:43.476 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (28m) 79s ago 28m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:58:43.476 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (28m) 79s ago 28m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:58:43.476 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (28m) 79s ago 28m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:58:43.476 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (27m) 79s ago 27m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:58:43.476 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (4m) 79s ago 27m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:58:43.476 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (25m) 63s ago 25m 76.5M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:58:43.476 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (25m) 79s ago 25m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:58:43.476 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:43 smithi055 ceph-mon[86116]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:43.477 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:43 smithi055 ceph-mon[88498]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:44.214 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-05-20T22:58:45.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:45 smithi055 ceph-mon[88498]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:45.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:45 smithi055 ceph-mon[88498]: from='client.54204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:45.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:45 smithi055 ceph-mon[86116]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:45.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:45 smithi055 ceph-mon[86116]: from='client.54204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:45.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:45 smithi138 ceph-mon[67823]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:45.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:45 smithi138 ceph-mon[67823]: from='client.54204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:46.383 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:58:47.192 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-05-20T22:58:47.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:47 smithi055 ceph-mon[86116]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:47.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:47 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/124929996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:47.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:58:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:58:47] "GET /metrics HTTP/1.1" 200 37939 "" "Prometheus/2.43.0" 2024-05-20T22:58:47.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:47 smithi055 ceph-mon[88498]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:47.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:47 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/124929996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:47.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:47 smithi138 ceph-mon[67823]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:47 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/124929996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:49.385 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)" 2024-05-20T22:58:49.385 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:49 smithi055 ceph-mon[86116]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:49.386 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:49 smithi055 ceph-mon[88498]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:49.408 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:49 smithi138 ceph-mon[67823]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:49.987 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-05-20T22:58:50.307 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:50 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/3607014956' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:50.307 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:50.307 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:50 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/3607014956' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:50.308 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:50.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:50 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/3607014956' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:58:50.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:58:51.410 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:51 smithi055 ceph-mon[86116]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:51.411 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:51 smithi055 ceph-mon[88498]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:51.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:51 smithi138 ceph-mon[67823]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:52.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:52 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:52.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:52 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:52.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:52 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:58:53.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:53 smithi055 ceph-mon[86116]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:53.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:53 smithi055 ceph-mon[86116]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:53.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:53 smithi055 ceph-mon[88498]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:53.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:53 smithi055 ceph-mon[88498]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:53.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:53 smithi138 ceph-mon[67823]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:53.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:53 smithi138 ceph-mon[67823]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:54.483 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:58:55.064 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade status' 2024-05-20T22:58:55.383 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:55 smithi055 ceph-mon[86116]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:55.383 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:55 smithi055 ceph-mon[88498]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:55.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:55 smithi138 ceph-mon[67823]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:57.147 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:58:57.148 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": null, 2024-05-20T22:58:57.148 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": false, 2024-05-20T22:58:57.148 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "", 2024-05-20T22:58:57.148 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T22:58:57.148 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": null, 2024-05-20T22:58:57.148 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "", 2024-05-20T22:58:57.148 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:58:57.148 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:58:57.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:57 smithi055 ceph-mon[86116]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:57.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:57 smithi055 ceph-mon[88498]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:57.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:58:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:58:57] "GET /metrics HTTP/1.1" 200 37938 "" "Prometheus/2.43.0" 2024-05-20T22:58:57.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:57 smithi138 ceph-mon[67823]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:58:57.753 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T22:58:58.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:58 smithi055 ceph-mon[86116]: from='client.54228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:58.347 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:58 smithi055 ceph-mon[88498]: from='client.54228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:58.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:58 smithi138 ceph-mon[67823]: from='client.54228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:58:59.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:58:59 smithi055 ceph-mon[86116]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:59.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:58:59 smithi055 ceph-mon[88498]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:59.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:58:59 smithi138 ceph-mon[67823]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:58:59.810 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T22:59:00.450 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:00 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/3182730203' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:59:00.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:00 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/3182730203' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:59:00.475 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-05-20T22:59:00.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:00 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/3182730203' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:59:01.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:01 smithi055 ceph-mon[86116]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:01.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:01 smithi055 ceph-mon[88498]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:01.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:01 smithi138 ceph-mon[67823]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:02.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:02 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:02.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:02 smithi055 ceph-mon[86116]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:02.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:02 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:02.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:02 smithi055 ceph-mon[88498]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:02.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:02 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:02.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:02 smithi138 ceph-mon[67823]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:03.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:03 smithi055 ceph-mon[86116]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:03.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:03 smithi055 ceph-mon[88498]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:03.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:03 smithi138 ceph-mon[67823]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:04.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:04 smithi055 ceph-mon[88498]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:04.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:04 smithi055 ceph-mon[86116]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:04.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:04 smithi138 ceph-mon[67823]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:04.702 INFO:teuthology.orchestra.run.smithi055.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:59:05.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:05 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:05.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:05 smithi055 ceph-mon[86116]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:59:05.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:05 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:05.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:05 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:05.386 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:05 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:05.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:05 smithi055 ceph-mon[88498]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:59:05.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:05 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:05.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:05 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:05.411 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-20T22:59:05.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:05 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:05.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:05 smithi138 ceph-mon[67823]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:59:05.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:05 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:05.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:05 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:06.378 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[86116]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:06.378 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:06.378 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:59:06.378 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:06.378 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[86116]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:59:06.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[88498]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:06.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:06.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:59:06.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:06.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:06 smithi055 ceph-mon[88498]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:59:06.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:06 smithi138 ceph-mon[67823]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:06.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:06 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:06.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:06 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:59:06.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:06 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:06.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:06 smithi138 ceph-mon[67823]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:59:07.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:59:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:59:07] "GET /metrics HTTP/1.1" 200 37940 "" "Prometheus/2.43.0" 2024-05-20T22:59:07.675 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (4m) 88s ago 26m 26.7M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (3m) 104s ago 26m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (3m) 88s ago 25m 50.4M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (5m) 104s ago 31m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (15m) 88s ago 32m 496M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (118s) 88s ago 32m 38.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (2m) 104s ago 31m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (93s) 88s ago 31m 20.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T22:59:08.372 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (5m) 88s ago 27m 15.6M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (5m) 104s ago 27m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (30m) 88s ago 30m 54.7M 2993M 17.2.0 e1d6a67b021e 2ee9c86427b4 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (30m) 88s ago 30m 55.9M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (30m) 88s ago 30m 54.0M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (29m) 88s ago 29m 53.6M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (29m) 104s ago 29m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (28m) 104s ago 28m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (28m) 104s ago 28m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (28m) 104s ago 28m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (4m) 104s ago 27m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (25m) 88s ago 25m 76.5M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:59:08.373 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (25m) 104s ago 25m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:59:08.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:08.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='client.54246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:08.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='client.64186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:08.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:08.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:08.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:08.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:08.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:59:08.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:08.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='client.54246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:08.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='client.64186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:08.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:08.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:08.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:08.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:08.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:59:08.803 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:59:08.804 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:59:08.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:59:08.804 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:59:08.804 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-20T22:59:08.804 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 5 2024-05-20T22:59:08.804 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:59:08.804 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:59:08.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='client.54246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='client.64186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:08.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:59:09.166 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:59:09.166 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:59:09.166 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:59:09.166 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-05-20T22:59:09.166 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T22:59:09.166 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "0/8 daemons upgraded", 2024-05-20T22:59:09.166 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:59:09.166 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:59:09.166 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:59:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:59:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:59:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mgr 2024-05-20T22:59:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mon 2024-05-20T22:59:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all crash 2024-05-20T22:59:09.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: Upgrade: osd.0 is safe to restart 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/1165177725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[86116]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mgr 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mon 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all crash 2024-05-20T22:59:09.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:59:09.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: Upgrade: osd.0 is safe to restart 2024-05-20T22:59:09.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/1165177725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:09.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:09 smithi055 ceph-mon[88498]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:09.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T22:59:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:59:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mgr 2024-05-20T22:59:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mon 2024-05-20T22:59:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all crash 2024-05-20T22:59:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:59:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: Upgrade: osd.0 is safe to restart 2024-05-20T22:59:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/1165177725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:09.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:09 smithi138 ceph-mon[67823]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:11.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:11 smithi138 ceph-mon[67823]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:11 smithi138 ceph-mon[67823]: Upgrade: Updating osd.0 2024-05-20T22:59:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:59:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:11 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:11.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:11 smithi138 ceph-mon[67823]: Deploying daemon osd.0 on smithi055 2024-05-20T22:59:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[86116]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[86116]: Upgrade: Updating osd.0 2024-05-20T22:59:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:59:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[86116]: Deploying daemon osd.0 on smithi055 2024-05-20T22:59:11.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[88498]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[88498]: Upgrade: Updating osd.0 2024-05-20T22:59:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:59:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:11.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:11 smithi055 ceph-mon[88498]: Deploying daemon osd.0 on smithi055 2024-05-20T22:59:12.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:12 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:12 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:12.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:12 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:13.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:13 smithi055 ceph-mon[86116]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:13.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:13 smithi055 ceph-mon[88498]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:13.564 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:13 smithi055 systemd[1]: Stopping Ceph osd.0 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:59:13.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:13 smithi138 ceph-mon[67823]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:13.982 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[32334]: 2024-05-20T22:59:13.570+0000 7ff184b6f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:59:13.983 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[32334]: 2024-05-20T22:59:13.570+0000 7ff184b6f700 -1 osd.0 114 *** Got signal Terminated *** 2024-05-20T22:59:13.983 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[32334]: 2024-05-20T22:59:13.570+0000 7ff184b6f700 -1 osd.0 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:59:14.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:14 smithi055 ceph-mon[86116]: osd.0 marked itself down and dead 2024-05-20T22:59:14.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:14 smithi055 ceph-mon[88498]: osd.0 marked itself down and dead 2024-05-20T22:59:14.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:14 smithi138 ceph-mon[67823]: osd.0 marked itself down and dead 2024-05-20T22:59:14.789 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:14 smithi055 podman[95339]: 2024-05-20 22:59:14.536087961 +0000 UTC m=+1.091086497 container died 2ee9c86427b418f4dfd7544038308342bca7cd7a8b61b1257e4fec7bdc90abdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, version=8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public) 2024-05-20T22:59:15.097 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:14 smithi055 podman[95339]: 2024-05-20 22:59:14.871932827 +0000 UTC m=+1.426931344 container cleanup 2ee9c86427b418f4dfd7544038308342bca7cd7a8b61b1257e4fec7bdc90abdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, distribution-scope=public, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_CLEAN=True, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T22:59:15.098 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:14 smithi055 bash[95339]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0 2024-05-20T22:59:15.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:15 smithi055 ceph-mon[88498]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:15.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:15 smithi055 ceph-mon[88498]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:59:15.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:15 smithi055 ceph-mon[88498]: osdmap e115: 8 total, 7 up, 8 in 2024-05-20T22:59:15.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:15 smithi055 ceph-mon[86116]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:15.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:15 smithi055 ceph-mon[86116]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:59:15.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:15 smithi055 ceph-mon[86116]: osdmap e115: 8 total, 7 up, 8 in 2024-05-20T22:59:15.484 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:15 smithi055 podman[95354]: 2024-05-20 22:59:15.106519518 +0000 UTC m=+0.567211901 container remove 2ee9c86427b418f4dfd7544038308342bca7cd7a8b61b1257e4fec7bdc90abdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0, maintainer=Guillaume Abrioux , name=centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_BRANCH=HEAD, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.openshift.expose-services=, architecture=x86_64, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, build-date=2022-05-03T08:36:31.336870) 2024-05-20T22:59:15.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:15 smithi138 ceph-mon[67823]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:15.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:15 smithi138 ceph-mon[67823]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:59:15.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:15 smithi138 ceph-mon[67823]: osdmap e115: 8 total, 7 up, 8 in 2024-05-20T22:59:15.818 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:15 smithi055 podman[95443]: 2024-05-20 22:59:15.475132977 +0000 UTC m=+0.018590119 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:59:15.818 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:15 smithi055 podman[95443]: 2024-05-20 22:59:15.599439026 +0000 UTC m=+0.142896148 container create 3a3f114cb14e0426863fed88f706953bbc350423b46d3be0fb6554267efa34bf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-05-20T22:59:15.818 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:15 smithi055 podman[95443]: 2024-05-20 22:59:15.827079401 +0000 UTC m=+0.370536536 container init 3a3f114cb14e0426863fed88f706953bbc350423b46d3be0fb6554267efa34bf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-20T22:59:16.128 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:15 smithi055 podman[95443]: 2024-05-20 22:59:15.83086327 +0000 UTC m=+0.374320401 container start 3a3f114cb14e0426863fed88f706953bbc350423b46d3be0fb6554267efa34bf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:59:16.128 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:15 smithi055 podman[95443]: 2024-05-20 22:59:15.902078331 +0000 UTC m=+0.445535468 container attach 3a3f114cb14e0426863fed88f706953bbc350423b46d3be0fb6554267efa34bf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2) 2024-05-20T22:59:16.388 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:16 smithi055 ceph-mon[86116]: osdmap e116: 8 total, 7 up, 8 in 2024-05-20T22:59:16.389 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:16 smithi055 ceph-mon[88498]: osdmap e116: 8 total, 7 up, 8 in 2024-05-20T22:59:16.389 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:16 smithi055 podman[95443]: 2024-05-20 22:59:16.137816664 +0000 UTC m=+0.681273803 container died 3a3f114cb14e0426863fed88f706953bbc350423b46d3be0fb6554267efa34bf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:59:16.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:16 smithi138 ceph-mon[67823]: osdmap e116: 8 total, 7 up, 8 in 2024-05-20T22:59:17.162 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:16 smithi055 podman[95469]: 2024-05-20 22:59:16.781404889 +0000 UTC m=+0.637616817 container remove 3a3f114cb14e0426863fed88f706953bbc350423b46d3be0fb6554267efa34bf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:59:17.163 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:16 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.0.service: Deactivated successfully. 2024-05-20T22:59:17.163 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:16 smithi055 systemd[1]: Stopped Ceph osd.0 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:59:17.163 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:16 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.0.service: Consumed 15.451s CPU time. 2024-05-20T22:59:17.163 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 systemd[1]: Starting Ceph osd.0 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:59:17.477 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:17 smithi055 ceph-mon[86116]: pgmap v63: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:17.477 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:17 smithi055 ceph-mon[86116]: Health check failed: Degraded data redundancy: 70/627 objects degraded (11.164%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:59:17.478 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:17 smithi055 ceph-mon[88498]: pgmap v63: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:17.478 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:17 smithi055 ceph-mon[88498]: Health check failed: Degraded data redundancy: 70/627 objects degraded (11.164%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:59:17.478 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:59:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:59:17] "GET /metrics HTTP/1.1" 200 37940 "" "Prometheus/2.43.0" 2024-05-20T22:59:17.478 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 podman[95581]: 2024-05-20 22:59:17.141342923 +0000 UTC m=+0.018244846 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:59:17.478 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 podman[95581]: 2024-05-20 22:59:17.24928376 +0000 UTC m=+0.126185690 container create 6d202ec4d941238edce54a251835c078d59a9245d2d841400a4db9f9f6f83fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514) 2024-05-20T22:59:17.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:17 smithi138 ceph-mon[67823]: pgmap v63: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:17.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:17 smithi138 ceph-mon[67823]: Health check failed: Degraded data redundancy: 70/627 objects degraded (11.164%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:59:17.733 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 podman[95581]: 2024-05-20 22:59:17.485897503 +0000 UTC m=+0.362799433 container init 6d202ec4d941238edce54a251835c078d59a9245d2d841400a4db9f9f6f83fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate, org.label-schema.build-date=20240514, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-05-20T22:59:17.733 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 podman[95581]: 2024-05-20 22:59:17.490171549 +0000 UTC m=+0.367073472 container start 6d202ec4d941238edce54a251835c078d59a9245d2d841400a4db9f9f6f83fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-05-20T22:59:17.733 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 podman[95581]: 2024-05-20 22:59:17.568621502 +0000 UTC m=+0.445523424 container attach 6d202ec4d941238edce54a251835c078d59a9245d2d841400a4db9f9f6f83fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514) 2024-05-20T22:59:18.233 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate[95597]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-20T22:59:18.233 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate[95597]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 bash[95581]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 bash[95581]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 bash[95581]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 bash[95581]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 bash[95581]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 bash[95581]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 bash[95581]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate[95597]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate[95597]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate[95597]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate[95597]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-20T22:59:18.234 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate[95597]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-20T22:59:18.235 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:17 smithi055 podman[95581]: 2024-05-20 22:59:17.982386171 +0000 UTC m=+0.859288096 container died 6d202ec4d941238edce54a251835c078d59a9245d2d841400a4db9f9f6f83fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:59:18.964 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:18 smithi055 podman[95732]: 2024-05-20 22:59:18.656285023 +0000 UTC m=+0.662587525 container remove 6d202ec4d941238edce54a251835c078d59a9245d2d841400a4db9f9f6f83fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514) 2024-05-20T22:59:18.965 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:18 smithi055 podman[95791]: 2024-05-20 22:59:18.87398686 +0000 UTC m=+0.109621033 container create ea50151bc837e0f0c72aca18bfe06aa974c1b496321694942faf9b39f38e1ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS) 2024-05-20T22:59:18.965 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:18 smithi055 podman[95791]: 2024-05-20 22:59:18.783081967 +0000 UTC m=+0.018716136 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:59:19.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:19 smithi055 ceph-mon[86116]: pgmap v64: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:19.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:19 smithi055 ceph-mon[88498]: pgmap v64: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:19.233 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:19 smithi055 podman[95791]: 2024-05-20 22:59:19.093226628 +0000 UTC m=+0.328860783 container init ea50151bc837e0f0c72aca18bfe06aa974c1b496321694942faf9b39f38e1ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0, ceph=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-20T22:59:19.233 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:19 smithi055 podman[95791]: 2024-05-20 22:59:19.096606635 +0000 UTC m=+0.332240790 container start ea50151bc837e0f0c72aca18bfe06aa974c1b496321694942faf9b39f38e1ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:59:19.233 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:19 smithi055 ceph-osd[95810]: -- 172.21.15.55:0/4229843414 <== mon.1 v2:172.21.15.55:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564486d91680 con 0x564486d5ac00 2024-05-20T22:59:19.233 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:19 smithi055 bash[95791]: ea50151bc837e0f0c72aca18bfe06aa974c1b496321694942faf9b39f38e1ba9 2024-05-20T22:59:19.233 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:19 smithi055 systemd[1]: Started Ceph osd.0 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:59:19.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:19 smithi138 ceph-mon[67823]: pgmap v64: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:19.982 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:19 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[95806]: 2024-05-20T22:59:19.642+0000 7f80784617c0 -1 Falling back to public interface 2024-05-20T22:59:20.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:20.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[88498]: pgmap v65: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:20.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:20.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:20.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[86116]: pgmap v65: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:20.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:20.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:20.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:20 smithi138 ceph-mon[67823]: pgmap v65: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:20.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:20.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:21.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:21 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:21.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:21 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:21.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:21 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:22.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:22 smithi055 ceph-mon[86116]: pgmap v66: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:22.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:22 smithi055 ceph-mon[88498]: pgmap v66: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:22.575 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[95806]: 2024-05-20T22:59:22.385+0000 7f80784617c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-05-20T22:59:22.575 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[95806]: 2024-05-20T22:59:22.416+0000 7f80784617c0 -1 osd.0 114 log_to_monitors true 2024-05-20T22:59:22.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:22 smithi138 ceph-mon[67823]: pgmap v66: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:23.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:23 smithi138 ceph-mon[67823]: from='osd.0 [v2:172.21.15.55:6802/1246236667,v1:172.21.15.55:6803/1246236667]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:59:23.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:23 smithi138 ceph-mon[67823]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:59:23.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:23 smithi055 ceph-mon[86116]: from='osd.0 [v2:172.21.15.55:6802/1246236667,v1:172.21.15.55:6803/1246236667]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:59:23.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:23 smithi055 ceph-mon[86116]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:59:23.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:23 smithi055 ceph-mon[88498]: from='osd.0 [v2:172.21.15.55:6802/1246236667,v1:172.21.15.55:6803/1246236667]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:59:23.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:23 smithi055 ceph-mon[88498]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:59:24.232 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 22:59:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[95806]: 2024-05-20T22:59:23.902+0000 7f80658b0700 -1 osd.0 114 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:59:24.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:24 smithi138 ceph-mon[67823]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:59:24.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:24 smithi138 ceph-mon[67823]: osdmap e117: 8 total, 7 up, 8 in 2024-05-20T22:59:24.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:24 smithi138 ceph-mon[67823]: from='osd.0 [v2:172.21.15.55:6802/1246236667,v1:172.21.15.55:6803/1246236667]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:24.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:24 smithi138 ceph-mon[67823]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:24.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:24 smithi138 ceph-mon[67823]: pgmap v68: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:24.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:24 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:24.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:24 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:24.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:24 smithi138 ceph-mon[67823]: from='osd.0 ' entity='osd.0' 2024-05-20T22:59:24.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[86116]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:59:24.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[86116]: osdmap e117: 8 total, 7 up, 8 in 2024-05-20T22:59:24.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[86116]: from='osd.0 [v2:172.21.15.55:6802/1246236667,v1:172.21.15.55:6803/1246236667]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:24.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[86116]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:24.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[86116]: pgmap v68: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:24.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:24.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:24.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[86116]: from='osd.0 ' entity='osd.0' 2024-05-20T22:59:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[88498]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:59:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[88498]: osdmap e117: 8 total, 7 up, 8 in 2024-05-20T22:59:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[88498]: from='osd.0 [v2:172.21.15.55:6802/1246236667,v1:172.21.15.55:6803/1246236667]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[88498]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[88498]: pgmap v68: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-05-20T22:59:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:24.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:24 smithi055 ceph-mon[88498]: from='osd.0 ' entity='osd.0' 2024-05-20T22:59:25.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[86116]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:59:25.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[86116]: osd.0 [v2:172.21.15.55:6802/1246236667,v1:172.21.15.55:6803/1246236667] boot 2024-05-20T22:59:25.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[86116]: osdmap e118: 8 total, 8 up, 8 in 2024-05-20T22:59:25.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:59:25.589 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:25.589 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:25.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[88498]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:59:25.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[88498]: osd.0 [v2:172.21.15.55:6802/1246236667,v1:172.21.15.55:6803/1246236667] boot 2024-05-20T22:59:25.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[88498]: osdmap e118: 8 total, 8 up, 8 in 2024-05-20T22:59:25.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:59:25.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:25.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:25 smithi138 ceph-mon[67823]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:59:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:25 smithi138 ceph-mon[67823]: osd.0 [v2:172.21.15.55:6802/1246236667,v1:172.21.15.55:6803/1246236667] boot 2024-05-20T22:59:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:25 smithi138 ceph-mon[67823]: osdmap e118: 8 total, 8 up, 8 in 2024-05-20T22:59:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:59:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:27.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:26 smithi055 ceph-mon[88498]: pgmap v70: 161 pgs: 37 peering, 9 active+undersized+degraded, 12 active+undersized, 103 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 31/627 objects degraded (4.944%) 2024-05-20T22:59:27.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:26 smithi055 ceph-mon[88498]: osdmap e119: 8 total, 8 up, 8 in 2024-05-20T22:59:27.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:26 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded (PG_DEGRADED) 2024-05-20T22:59:27.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:26 smithi055 ceph-mon[86116]: pgmap v70: 161 pgs: 37 peering, 9 active+undersized+degraded, 12 active+undersized, 103 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 31/627 objects degraded (4.944%) 2024-05-20T22:59:27.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:26 smithi055 ceph-mon[86116]: osdmap e119: 8 total, 8 up, 8 in 2024-05-20T22:59:27.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:26 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded (PG_DEGRADED) 2024-05-20T22:59:27.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:26 smithi138 ceph-mon[67823]: pgmap v70: 161 pgs: 37 peering, 9 active+undersized+degraded, 12 active+undersized, 103 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 31/627 objects degraded (4.944%) 2024-05-20T22:59:27.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:26 smithi138 ceph-mon[67823]: osdmap e119: 8 total, 8 up, 8 in 2024-05-20T22:59:27.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:26 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded (PG_DEGRADED) 2024-05-20T22:59:27.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:59:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:59:27] "GET /metrics HTTP/1.1" 200 38073 "" "Prometheus/2.43.0" 2024-05-20T22:59:28.918 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:59:28 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T22:59:28.693+0000 7f06747a2700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: pgmap v72: 161 pgs: 37 peering, 9 active+undersized+degraded, 12 active+undersized, 103 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 31/627 objects degraded (4.944%) 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: pgmap v72: 161 pgs: 37 peering, 9 active+undersized+degraded, 12 active+undersized, 103 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 31/627 objects degraded (4.944%) 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:29.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:29.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:29.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:29.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: pgmap v72: 161 pgs: 37 peering, 9 active+undersized+degraded, 12 active+undersized, 103 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 31/627 objects degraded (4.944%) 2024-05-20T22:59:29.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:29.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:29.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:29.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:59:29.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:29.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:29.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:29.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:29.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:29.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:30.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:29 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:30.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:29 smithi055 ceph-mon[86116]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-05-20T22:59:30.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:29 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:30.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:29 smithi055 ceph-mon[88498]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-05-20T22:59:30.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:29 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:30.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:29 smithi138 ceph-mon[67823]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-05-20T22:59:31.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:30 smithi055 ceph-mon[86116]: pgmap v73: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-05-20T22:59:31.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:30 smithi055 ceph-mon[86116]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded) 2024-05-20T22:59:31.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:30 smithi055 ceph-mon[86116]: Cluster is now healthy 2024-05-20T22:59:31.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:30 smithi055 ceph-mon[88498]: pgmap v73: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-05-20T22:59:31.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:30 smithi055 ceph-mon[88498]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded) 2024-05-20T22:59:31.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:30 smithi055 ceph-mon[88498]: Cluster is now healthy 2024-05-20T22:59:31.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:30 smithi138 ceph-mon[67823]: pgmap v73: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-05-20T22:59:31.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:30 smithi138 ceph-mon[67823]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded) 2024-05-20T22:59:31.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:30 smithi138 ceph-mon[67823]: Cluster is now healthy 2024-05-20T22:59:32.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:31 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:32.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:31 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:32.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:31 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:33.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:32 smithi055 ceph-mon[86116]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:33.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:32 smithi055 ceph-mon[88498]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:33.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:32 smithi138 ceph-mon[67823]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:34 smithi055 ceph-mon[88498]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:35.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:34 smithi055 ceph-mon[86116]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:35.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:35.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:35.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:34 smithi138 ceph-mon[67823]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:35.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:35.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:37.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:36 smithi055 ceph-mon[86116]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T22:59:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:36 smithi055 ceph-mon[88498]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T22:59:37.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:36 smithi138 ceph-mon[67823]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T22:59:37.510 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:59:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:59:37] "GET /metrics HTTP/1.1" 200 38087 "" "Prometheus/2.43.0" 2024-05-20T22:59:39.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:38 smithi055 ceph-mon[86116]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-05-20T22:59:39.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:38 smithi055 ceph-mon[88498]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-05-20T22:59:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:38 smithi138 ceph-mon[67823]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-05-20T22:59:39.544 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T22:59:40.228 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:59:40.228 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (4m) 16s ago 26m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T22:59:40.228 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (4m) 2m ago 26m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T22:59:40.228 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (4m) 16s ago 26m 58.1M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T22:59:40.228 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (6m) 2m ago 31m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T22:59:40.228 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (16m) 16s ago 33m 505M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T22:59:40.228 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (2m) 16s ago 33m 45.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T22:59:40.228 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (3m) 2m ago 31m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (2m) 16s ago 32m 36.8M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (5m) 16s ago 27m 16.2M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (5m) 2m ago 27m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (21s) 16s ago 31m 12.7M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (31m) 16s ago 30m 55.4M 2993M 17.2.0 e1d6a67b021e c1c3aa905701 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (30m) 16s ago 30m 54.5M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (30m) 16s ago 30m 54.5M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (29m) 2m ago 29m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (29m) 2m ago 29m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (29m) 2m ago 29m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (28m) 2m ago 28m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (5m) 2m ago 27m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (26m) 16s ago 26m 76.9M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T22:59:40.229 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (26m) 2m ago 26m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T22:59:40.647 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:59:40.647 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T22:59:40.647 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T22:59:40.647 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:59:40.647 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T22:59:40.647 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:59:40.647 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 6 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T22:59:40.648 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:59:40.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:40 smithi138 ceph-mon[67823]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:40 smithi138 ceph-mon[67823]: from='client.64216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:40 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:40.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:40 smithi138 ceph-mon[67823]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:40.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:40 smithi055 ceph-mon[86116]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:40.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:40 smithi055 ceph-mon[86116]: from='client.64216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:40.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:40 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:40.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:40 smithi055 ceph-mon[86116]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:40.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:40 smithi055 ceph-mon[88498]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:40.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:40 smithi055 ceph-mon[88498]: from='client.64216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:40.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:40 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:40.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:40 smithi055 ceph-mon[88498]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:40.999 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T22:59:41.000 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:59:41.000 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T22:59:41.000 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-05-20T22:59:41.000 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T22:59:41.000 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "1/8 daemons upgraded", 2024-05-20T22:59:41.000 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:59:41.001 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T22:59:41.001 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T22:59:41.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:41 smithi138 ceph-mon[67823]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:41.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:41 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:41.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:41 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/249312059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:41.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:41 smithi138 ceph-mon[67823]: from='client.64240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:41 smithi055 ceph-mon[86116]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:41 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:41 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/249312059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:41.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:41 smithi055 ceph-mon[86116]: from='client.64240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:41.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:41 smithi055 ceph-mon[88498]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:41.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:41 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:41.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:41 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/249312059' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:59:41.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:41 smithi055 ceph-mon[88498]: from='client.64240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:59:42.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:42 smithi138 ceph-mon[67823]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:42.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:42 smithi055 ceph-mon[86116]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:42.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:42 smithi055 ceph-mon[88498]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:44.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:44 smithi055 ceph-mon[86116]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:44.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:44 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:44.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:44 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:44.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:44 smithi055 ceph-mon[86116]: Upgrade: osd.1 is safe to restart 2024-05-20T22:59:44.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:44 smithi055 ceph-mon[88498]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:44.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:44 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:44.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:44 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:44.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:44 smithi055 ceph-mon[88498]: Upgrade: osd.1 is safe to restart 2024-05-20T22:59:44.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:44 smithi138 ceph-mon[67823]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:44.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:44 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:44.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:44 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:59:44.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:44 smithi138 ceph-mon[67823]: Upgrade: osd.1 is safe to restart 2024-05-20T22:59:46.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[86116]: Upgrade: Updating osd.1 2024-05-20T22:59:46.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:46.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:59:46.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:46.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[86116]: Deploying daemon osd.1 on smithi055 2024-05-20T22:59:46.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[86116]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:46.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[88498]: Upgrade: Updating osd.1 2024-05-20T22:59:46.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:46.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:59:46.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:46.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[88498]: Deploying daemon osd.1 on smithi055 2024-05-20T22:59:46.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:46 smithi055 ceph-mon[88498]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:46.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:46 smithi138 ceph-mon[67823]: Upgrade: Updating osd.1 2024-05-20T22:59:46.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:46 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:46.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:46 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:59:46.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:46 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:59:46.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:46 smithi138 ceph-mon[67823]: Deploying daemon osd.1 on smithi055 2024-05-20T22:59:46.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:46 smithi138 ceph-mon[67823]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T22:59:47.513 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:59:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:59:47] "GET /metrics HTTP/1.1" 200 38087 "" "Prometheus/2.43.0" 2024-05-20T22:59:48.691 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:48 smithi055 ceph-mon[86116]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:48.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:48 smithi055 ceph-mon[88498]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:48.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:48 smithi138 ceph-mon[67823]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:48.973 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:48 smithi055 systemd[1]: Stopping Ceph osd.1 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:59:49.232 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[35281]: 2024-05-20T22:59:48.980+0000 7f9395924700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:59:49.233 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[35281]: 2024-05-20T22:59:48.980+0000 7f9395924700 -1 osd.1 119 *** Got signal Terminated *** 2024-05-20T22:59:49.233 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:48 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[35281]: 2024-05-20T22:59:48.980+0000 7f9395924700 -1 osd.1 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:59:49.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:49 smithi138 ceph-mon[67823]: osd.1 marked itself down and dead 2024-05-20T22:59:49.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:49.901 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:49 smithi055 ceph-mon[86116]: osd.1 marked itself down and dead 2024-05-20T22:59:49.901 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:49.902 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:49 smithi055 ceph-mon[88498]: osd.1 marked itself down and dead 2024-05-20T22:59:49.902 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:59:50.233 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:50 smithi055 podman[100492]: 2024-05-20 22:59:50.068379562 +0000 UTC m=+1.210987983 container died c1c3aa9057015afe909c3cb4c77bee838185d590b48410065eb9c91a62ba0b78 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.openshift.expose-services=, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, ceph=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T22:59:50.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:50 smithi055 ceph-mon[86116]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:50.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:50 smithi055 ceph-mon[86116]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:59:50.698 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:50 smithi055 ceph-mon[86116]: osdmap e120: 8 total, 7 up, 8 in 2024-05-20T22:59:50.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:50 smithi055 ceph-mon[88498]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:50.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:50 smithi055 ceph-mon[88498]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:59:50.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:50 smithi055 ceph-mon[88498]: osdmap e120: 8 total, 7 up, 8 in 2024-05-20T22:59:50.699 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:50 smithi055 podman[100492]: 2024-05-20 22:59:50.438853601 +0000 UTC m=+1.581462002 container cleanup c1c3aa9057015afe909c3cb4c77bee838185d590b48410065eb9c91a62ba0b78 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, build-date=2022-05-03T08:36:31.336870, version=8, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_BRANCH=HEAD, architecture=x86_64, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T22:59:50.699 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:50 smithi055 bash[100492]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1 2024-05-20T22:59:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:50 smithi138 ceph-mon[67823]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T22:59:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:50 smithi138 ceph-mon[67823]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:59:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:50 smithi138 ceph-mon[67823]: osdmap e120: 8 total, 7 up, 8 in 2024-05-20T22:59:50.982 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:50 smithi055 podman[100520]: 2024-05-20 22:59:50.707359119 +0000 UTC m=+0.631121871 container remove c1c3aa9057015afe909c3cb4c77bee838185d590b48410065eb9c91a62ba0b78 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.openshift.expose-services=, distribution-scope=public, release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, architecture=x86_64, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-20T22:59:51.385 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:51 smithi055 podman[100610]: 2024-05-20 22:59:50.986509453 +0000 UTC m=+0.017911086 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:59:51.385 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:51 smithi055 podman[100610]: 2024-05-20 22:59:51.100126178 +0000 UTC m=+0.131527854 container create e96c3279bb7fce76982b00057403908896396ae396ca660f561bfabc3669c221 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-20T22:59:51.385 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:51 smithi055 podman[100610]: 2024-05-20 22:59:51.311226672 +0000 UTC m=+0.342628324 container init e96c3279bb7fce76982b00057403908896396ae396ca660f561bfabc3669c221 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:59:51.385 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:51 smithi055 podman[100610]: 2024-05-20 22:59:51.314771613 +0000 UTC m=+0.346173248 container start e96c3279bb7fce76982b00057403908896396ae396ca660f561bfabc3669c221 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:59:51.386 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:51 smithi055 podman[100610]: 2024-05-20 22:59:51.394034054 +0000 UTC m=+0.425435703 container attach e96c3279bb7fce76982b00057403908896396ae396ca660f561bfabc3669c221 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2) 2024-05-20T22:59:51.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:51 smithi055 ceph-mon[86116]: osdmap e121: 8 total, 7 up, 8 in 2024-05-20T22:59:51.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:51 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:51.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:51 smithi055 ceph-mon[88498]: osdmap e121: 8 total, 7 up, 8 in 2024-05-20T22:59:51.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:51 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:51.733 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:51 smithi055 podman[100610]: 2024-05-20 22:59:51.622482238 +0000 UTC m=+0.653883872 container died e96c3279bb7fce76982b00057403908896396ae396ca660f561bfabc3669c221 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-20T22:59:51.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:51 smithi138 ceph-mon[67823]: osdmap e121: 8 total, 7 up, 8 in 2024-05-20T22:59:51.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:51 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T22:59:52.158 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 podman[100636]: 2024-05-20 22:59:52.164702586 +0000 UTC m=+0.536173102 container remove e96c3279bb7fce76982b00057403908896396ae396ca660f561bfabc3669c221 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-05-20T22:59:52.483 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.1.service: Deactivated successfully. 2024-05-20T22:59:52.483 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 systemd[1]: Stopped Ceph osd.1 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:59:52.483 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.1.service: Consumed 16.349s CPU time. 2024-05-20T22:59:52.483 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 systemd[1]: Starting Ceph osd.1 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T22:59:52.852 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:52 smithi055 ceph-mon[86116]: pgmap v86: 161 pgs: 56 peering, 1 stale+active+clean, 104 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:59:52.852 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:52 smithi055 ceph-mon[86116]: Health check failed: Reduced data availability: 11 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2024-05-20T22:59:52.852 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:52 smithi055 ceph-mon[88498]: pgmap v86: 161 pgs: 56 peering, 1 stale+active+clean, 104 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:59:52.852 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:52 smithi055 ceph-mon[88498]: Health check failed: Reduced data availability: 11 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2024-05-20T22:59:52.853 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 podman[100739]: 2024-05-20 22:59:52.525107352 +0000 UTC m=+0.017944784 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:59:52.853 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 podman[100739]: 2024-05-20 22:59:52.640885298 +0000 UTC m=+0.133722709 container create 9997898af8b7e141be53bd58b47d4828aa2b3d9d8b9cd039f4c9fef1e0934338 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux ) 2024-05-20T22:59:52.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:52 smithi138 ceph-mon[67823]: pgmap v86: 161 pgs: 56 peering, 1 stale+active+clean, 104 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:59:52.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:52 smithi138 ceph-mon[67823]: Health check failed: Reduced data availability: 11 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2024-05-20T22:59:53.233 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 podman[100739]: 2024-05-20 22:59:52.861150869 +0000 UTC m=+0.353988283 container init 9997898af8b7e141be53bd58b47d4828aa2b3d9d8b9cd039f4c9fef1e0934338 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True) 2024-05-20T22:59:53.233 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 podman[100739]: 2024-05-20 22:59:52.864763593 +0000 UTC m=+0.357601007 container start 9997898af8b7e141be53bd58b47d4828aa2b3d9d8b9cd039f4c9fef1e0934338 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2) 2024-05-20T22:59:53.233 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:52 smithi055 podman[100739]: 2024-05-20 22:59:52.943525527 +0000 UTC m=+0.436362944 container attach 9997898af8b7e141be53bd58b47d4828aa2b3d9d8b9cd039f4c9fef1e0934338 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate[100755]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate[100755]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 bash[100739]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 bash[100739]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 bash[100739]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 bash[100739]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 bash[100739]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 bash[100739]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 bash[100739]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate[100755]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate[100755]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-20T22:59:53.599 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate[100755]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-20T22:59:53.600 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate[100755]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-20T22:59:53.600 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate[100755]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-20T22:59:53.600 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:53 smithi055 podman[100739]: 2024-05-20 22:59:53.352081078 +0000 UTC m=+0.844918492 container died 9997898af8b7e141be53bd58b47d4828aa2b3d9d8b9cd039f4c9fef1e0934338 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-20T22:59:53.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:53 smithi138 ceph-mon[67823]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-05-20T22:59:53.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:53 smithi055 ceph-mon[86116]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-05-20T22:59:53.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:53 smithi055 ceph-mon[88498]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-05-20T22:59:54.429 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:54 smithi055 podman[100885]: 2024-05-20 22:59:54.106591076 +0000 UTC m=+0.748484308 container remove 9997898af8b7e141be53bd58b47d4828aa2b3d9d8b9cd039f4c9fef1e0934338 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514) 2024-05-20T22:59:54.429 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:54 smithi055 podman[100943]: 2024-05-20 22:59:54.234615272 +0000 UTC m=+0.018297041 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:59:54.430 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:54 smithi055 podman[100943]: 2024-05-20 22:59:54.332468378 +0000 UTC m=+0.116150134 container create eb6928b01e1a6c2af76c263690046b5f8f98b06e7e501358991787da1a18429d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:59:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:54 smithi138 ceph-mon[67823]: pgmap v87: 161 pgs: 1 active+undersized+degraded, 56 peering, 6 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 6/627 objects degraded (0.957%) 2024-05-20T22:59:54.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:54 smithi055 ceph-mon[86116]: pgmap v87: 161 pgs: 1 active+undersized+degraded, 56 peering, 6 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 6/627 objects degraded (0.957%) 2024-05-20T22:59:54.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:54 smithi055 ceph-mon[88498]: pgmap v87: 161 pgs: 1 active+undersized+degraded, 56 peering, 6 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 6/627 objects degraded (0.957%) 2024-05-20T22:59:54.983 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:54 smithi055 podman[100943]: 2024-05-20 22:59:54.543250093 +0000 UTC m=+0.326931850 container init eb6928b01e1a6c2af76c263690046b5f8f98b06e7e501358991787da1a18429d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux ) 2024-05-20T22:59:54.983 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:54 smithi055 podman[100943]: 2024-05-20 22:59:54.54664159 +0000 UTC m=+0.330323350 container start eb6928b01e1a6c2af76c263690046b5f8f98b06e7e501358991787da1a18429d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5) 2024-05-20T22:59:54.983 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:54 smithi055 ceph-osd[100961]: -- 172.21.15.55:0/813486339 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c7e0ac7680 con 0x55c7e0a90c00 2024-05-20T22:59:54.984 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:54 smithi055 bash[100943]: eb6928b01e1a6c2af76c263690046b5f8f98b06e7e501358991787da1a18429d 2024-05-20T22:59:54.984 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:54 smithi055 systemd[1]: Started Ceph osd.1 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T22:59:55.483 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:55 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[100957]: 2024-05-20T22:59:55.090+0000 7fde23d817c0 -1 Falling back to public interface 2024-05-20T22:59:55.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:55 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:55.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:55 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:55.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:55 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:55.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:55 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:55.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:55 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:55.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:55 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:55.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:55 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:55.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:55 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:56.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:55 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:56.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:55 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:56.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:55 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:56.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:55 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:59:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:56 smithi055 ceph-mon[86116]: pgmap v88: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-20T22:59:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:56 smithi055 ceph-mon[86116]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 17 pgs peering) 2024-05-20T22:59:56.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:56 smithi055 ceph-mon[88498]: pgmap v88: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-20T22:59:56.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:56 smithi055 ceph-mon[88498]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 17 pgs peering) 2024-05-20T22:59:57.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:56 smithi138 ceph-mon[67823]: pgmap v88: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-20T22:59:57.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:56 smithi138 ceph-mon[67823]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 17 pgs peering) 2024-05-20T22:59:57.482 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 22:59:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:22:59:57] "GET /metrics HTTP/1.1" 200 38090 "" "Prometheus/2.43.0" 2024-05-20T22:59:58.232 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[100957]: 2024-05-20T22:59:57.833+0000 7fde23d817c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-05-20T22:59:58.233 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[100957]: 2024-05-20T22:59:57.864+0000 7fde23d817c0 -1 osd.1 119 log_to_monitors true 2024-05-20T22:59:58.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:58 smithi055 ceph-mon[86116]: pgmap v89: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-20T22:59:58.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:58 smithi055 ceph-mon[86116]: from='osd.1 [v2:172.21.15.55:6810/1653868181,v1:172.21.15.55:6811/1653868181]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:59:58.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:58 smithi055 ceph-mon[86116]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:59:58.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:58 smithi055 ceph-mon[88498]: pgmap v89: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-20T22:59:58.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:58 smithi055 ceph-mon[88498]: from='osd.1 [v2:172.21.15.55:6810/1653868181,v1:172.21.15.55:6811/1653868181]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:59:58.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:58 smithi055 ceph-mon[88498]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:59:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:58 smithi138 ceph-mon[67823]: pgmap v89: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-20T22:59:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:58 smithi138 ceph-mon[67823]: from='osd.1 [v2:172.21.15.55:6810/1653868181,v1:172.21.15.55:6811/1653868181]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:59:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:58 smithi138 ceph-mon[67823]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:59:59.482 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 22:59:59 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[100957]: 2024-05-20T22:59:59.114+0000 7fde111d0700 -1 osd.1 119 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:59:59.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[88498]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T22:59:59.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[88498]: osdmap e122: 8 total, 7 up, 8 in 2024-05-20T22:59:59.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[88498]: from='osd.1 [v2:172.21.15.55:6810/1653868181,v1:172.21.15.55:6811/1653868181]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:59.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[88498]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:59.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:59.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:59.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[88498]: from='osd.1 ' entity='osd.1' 2024-05-20T22:59:59.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[86116]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T22:59:59.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[86116]: osdmap e122: 8 total, 7 up, 8 in 2024-05-20T22:59:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[86116]: from='osd.1 [v2:172.21.15.55:6810/1653868181,v1:172.21.15.55:6811/1653868181]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[86116]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T22:59:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T22:59:59.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 22:59:59 smithi055 ceph-mon[86116]: from='osd.1 ' entity='osd.1' 2024-05-20T23:00:00.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:59 smithi138 ceph-mon[67823]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T23:00:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:59 smithi138 ceph-mon[67823]: osdmap e122: 8 total, 7 up, 8 in 2024-05-20T23:00:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:59 smithi138 ceph-mon[67823]: from='osd.1 [v2:172.21.15.55:6810/1653868181,v1:172.21.15.55:6811/1653868181]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:00:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:59 smithi138 ceph-mon[67823]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:00:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:59 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:59 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:00.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 22:59:59 smithi138 ceph-mon[67823]: from='osd.1 ' entity='osd.1' 2024-05-20T23:00:00.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pgmap v91: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s; 84/627 objects degraded (13.397%) 2024-05-20T23:00:00.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 84/627 objects degraded (13.397%), 22 pgs degraded 2024-05-20T23:00:00.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: [WRN] OSD_DOWN: 1 osds down 2024-05-20T23:00:00.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: osd.1 (root=default,host=smithi055) is down 2024-05-20T23:00:00.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: [WRN] PG_DEGRADED: Degraded data redundancy: 84/627 objects degraded (13.397%), 22 pgs degraded 2024-05-20T23:00:00.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-05-20T23:00:00.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-05-20T23:00:00.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-05-20T23:00:00.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 2.a is active+undersized+degraded, acting [3,7] 2024-05-20T23:00:00.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.a is active+undersized+degraded, acting [6,4] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-05-20T23:00:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 4.9 is active+undersized+degraded, acting [4,3] 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 4.d is active+undersized+degraded, acting [4,2] 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 4.e is active+undersized+degraded, acting [4,6] 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 4.1f is active+undersized+degraded, acting [6,5] 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 5.c is active+undersized+degraded, acting [4,0] 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: pg 6.1a is active+undersized+degraded, acting [4,5] 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: osd.1 [v2:172.21.15.55:6810/1653868181,v1:172.21.15.55:6811/1653868181] boot 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: osdmap e123: 8 total, 8 up, 8 in 2024-05-20T23:00:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T23:00:00.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:00.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:00.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 84/627 objects degraded (13.397%), 22 pgs degraded (PG_DEGRADED) 2024-05-20T23:00:00.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pgmap v91: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s; 84/627 objects degraded (13.397%) 2024-05-20T23:00:00.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 84/627 objects degraded (13.397%), 22 pgs degraded 2024-05-20T23:00:00.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: [WRN] OSD_DOWN: 1 osds down 2024-05-20T23:00:00.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: osd.1 (root=default,host=smithi055) is down 2024-05-20T23:00:00.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: [WRN] PG_DEGRADED: Degraded data redundancy: 84/627 objects degraded (13.397%), 22 pgs degraded 2024-05-20T23:00:00.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 2.a is active+undersized+degraded, acting [3,7] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.a is active+undersized+degraded, acting [6,4] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-05-20T23:00:00.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 4.9 is active+undersized+degraded, acting [4,3] 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 4.d is active+undersized+degraded, acting [4,2] 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 4.e is active+undersized+degraded, acting [4,6] 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 4.1f is active+undersized+degraded, acting [6,5] 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 5.c is active+undersized+degraded, acting [4,0] 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: pg 6.1a is active+undersized+degraded, acting [4,5] 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: osd.1 [v2:172.21.15.55:6810/1653868181,v1:172.21.15.55:6811/1653868181] boot 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: osdmap e123: 8 total, 8 up, 8 in 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:00.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:00.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:00 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 84/627 objects degraded (13.397%), 22 pgs degraded (PG_DEGRADED) 2024-05-20T23:00:01.011 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pgmap v91: 161 pgs: 22 active+undersized+degraded, 41 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s; 84/627 objects degraded (13.397%) 2024-05-20T23:00:01.011 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 84/627 objects degraded (13.397%), 22 pgs degraded 2024-05-20T23:00:01.011 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: [WRN] OSD_DOWN: 1 osds down 2024-05-20T23:00:01.011 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: osd.1 (root=default,host=smithi055) is down 2024-05-20T23:00:01.011 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: [WRN] PG_DEGRADED: Degraded data redundancy: 84/627 objects degraded (13.397%), 22 pgs degraded 2024-05-20T23:00:01.011 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-05-20T23:00:01.011 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-05-20T23:00:01.011 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-05-20T23:00:01.011 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 2.a is active+undersized+degraded, acting [3,7] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.a is active+undersized+degraded, acting [6,4] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 4.9 is active+undersized+degraded, acting [4,3] 2024-05-20T23:00:01.012 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 4.d is active+undersized+degraded, acting [4,2] 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 4.e is active+undersized+degraded, acting [4,6] 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 4.1f is active+undersized+degraded, acting [6,5] 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 5.c is active+undersized+degraded, acting [4,0] 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: pg 6.1a is active+undersized+degraded, acting [4,5] 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: osd.1 [v2:172.21.15.55:6810/1653868181,v1:172.21.15.55:6811/1653868181] boot 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: osdmap e123: 8 total, 8 up, 8 in 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:01.013 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:00 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 84/627 objects degraded (13.397%), 22 pgs degraded (PG_DEGRADED) 2024-05-20T23:00:02.197 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:02 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:02.197 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:02 smithi055 ceph-mon[86116]: osdmap e124: 8 total, 8 up, 8 in 2024-05-20T23:00:02.197 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:02 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:02.198 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:02 smithi055 ceph-mon[88498]: osdmap e124: 8 total, 8 up, 8 in 2024-05-20T23:00:02.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:02 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:02.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:02 smithi138 ceph-mon[67823]: osdmap e124: 8 total, 8 up, 8 in 2024-05-20T23:00:03.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:03 smithi055 ceph-mon[88498]: pgmap v94: 161 pgs: 37 peering, 7 active+undersized+degraded, 19 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 35/627 objects degraded (5.582%) 2024-05-20T23:00:03.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:03 smithi055 ceph-mon[86116]: pgmap v94: 161 pgs: 37 peering, 7 active+undersized+degraded, 19 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 35/627 objects degraded (5.582%) 2024-05-20T23:00:03.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:03 smithi138 ceph-mon[67823]: pgmap v94: 161 pgs: 37 peering, 7 active+undersized+degraded, 19 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 35/627 objects degraded (5.582%) 2024-05-20T23:00:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: pgmap v95: 161 pgs: 37 peering, 6 active+undersized+degraded, 14 active+undersized, 104 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 29/627 objects degraded (4.625%) 2024-05-20T23:00:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: Upgrade: Finalizing container_image settings 2024-05-20T23:00:05.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:00:05.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:00:05.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:00:05.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:00:05.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:00:05.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:00:05.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:00:05.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: pgmap v95: 161 pgs: 37 peering, 6 active+undersized+degraded, 14 active+undersized, 104 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 29/627 objects degraded (4.625%) 2024-05-20T23:00:05.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:05.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:05.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:05.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: Upgrade: Finalizing container_image settings 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:00:05.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:00:05.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: Upgrade: Complete! 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:05.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:05.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:05.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:05.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:05.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:00:05.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:00:05.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: Upgrade: Complete! 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:00:05.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:00:05.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:05.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:05.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:05.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:05.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:05.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:05.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:05.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:05.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: pgmap v95: 161 pgs: 37 peering, 6 active+undersized+degraded, 14 active+undersized, 104 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 29/627 objects degraded (4.625%) 2024-05-20T23:00:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: Upgrade: Finalizing container_image settings 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:00:05.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:00:05.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: Upgrade: Complete! 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:05.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:05.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:06.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:05 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded (PG_DEGRADED) 2024-05-20T23:00:06.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:05 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded (PG_DEGRADED) 2024-05-20T23:00:06.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:05 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded (PG_DEGRADED) 2024-05-20T23:00:07.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:06 smithi055 ceph-mon[86116]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-20T23:00:07.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:06 smithi055 ceph-mon[86116]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded) 2024-05-20T23:00:07.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:06 smithi055 ceph-mon[86116]: Cluster is now healthy 2024-05-20T23:00:07.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:06 smithi055 ceph-mon[88498]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-20T23:00:07.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:06 smithi055 ceph-mon[88498]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded) 2024-05-20T23:00:07.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:06 smithi055 ceph-mon[88498]: Cluster is now healthy 2024-05-20T23:00:07.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:06 smithi138 ceph-mon[67823]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-20T23:00:07.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:06 smithi138 ceph-mon[67823]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded) 2024-05-20T23:00:07.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:06 smithi138 ceph-mon[67823]: Cluster is now healthy 2024-05-20T23:00:07.534 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:00:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:00:07] "GET /metrics HTTP/1.1" 200 38099 "" "Prometheus/2.43.0" 2024-05-20T23:00:09.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:08 smithi055 ceph-mon[86116]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:09.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:08 smithi055 ceph-mon[88498]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:09.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:08 smithi138 ceph-mon[67823]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:11.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:10 smithi055 ceph-mon[86116]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T23:00:11.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:10 smithi055 ceph-mon[88498]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T23:00:11.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:10 smithi138 ceph-mon[67823]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T23:00:11.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:11 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:11.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:11 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:12.015 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T23:00:12.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:11 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:13.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:12 smithi055 ceph-mon[86116]: from='client.54318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:13.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:12 smithi055 ceph-mon[86116]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-05-20T23:00:13.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:12 smithi055 ceph-mon[88498]: from='client.54318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:13.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:12 smithi055 ceph-mon[88498]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-05-20T23:00:13.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:12 smithi138 ceph-mon[67823]: from='client.54318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:13.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:12 smithi138 ceph-mon[67823]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-05-20T23:00:14.017 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:00:14.017 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (5m) 14s ago 27m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:00:14.017 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (5m) 2m ago 27m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:00:14.017 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (4m) 14s ago 26m 60.2M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:00:14.017 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (6m) 2m ago 32m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:00:14.017 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (16m) 14s ago 33m 506M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:00:14.017 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (3m) 14s ago 34m 46.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:00:14.017 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (3m) 2m ago 32m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (2m) 14s ago 32m 37.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (6m) 14s ago 28m 16.0M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (6m) 2m ago 28m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (55s) 14s ago 31m 50.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (19s) 14s ago 31m 12.0M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (31m) 14s ago 31m 55.3M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (30m) 14s ago 30m 55.1M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (30m) 2m ago 30m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (29m) 2m ago 29m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (29m) 2m ago 29m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (29m) 2m ago 29m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T23:00:14.018 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (5m) 2m ago 28m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:00:14.019 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (26m) 14s ago 26m 77.0M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:00:14.019 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (26m) 2m ago 26m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:00:14.662 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-05-20T23:00:15.210 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:14 smithi055 ceph-mon[86116]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:15.210 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:14 smithi055 ceph-mon[86116]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:15.210 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:14 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:15.211 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:14 smithi055 ceph-mon[88498]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:15.211 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:14 smithi055 ceph-mon[88498]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:15.211 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:14 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:15.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:14 smithi138 ceph-mon[67823]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:15.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:14 smithi138 ceph-mon[67823]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:15.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:14 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:16.749 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:00:17.056 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:16 smithi055 ceph-mon[86116]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:17.056 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:16 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/313977284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:17.056 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:16 smithi055 ceph-mon[88498]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:17.056 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:16 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/313977284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:17.347 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:00:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:00:17] "GET /metrics HTTP/1.1" 200 38099 "" "Prometheus/2.43.0" 2024-05-20T23:00:17.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:16 smithi138 ceph-mon[67823]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:17.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:16 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/313977284' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:17.373 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-05-20T23:00:19.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:18 smithi055 ceph-mon[86116]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:19.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:18 smithi055 ceph-mon[88498]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:19.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:18 smithi138 ceph-mon[67823]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:20.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:20 smithi055 ceph-mon[88498]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:20.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:20 smithi055 ceph-mon[88498]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:20.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:20.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:20 smithi055 ceph-mon[86116]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:20 smithi055 ceph-mon[86116]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:20.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:20.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:20 smithi138 ceph-mon[67823]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:20 smithi138 ceph-mon[67823]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:21.785 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:21 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:21.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:21 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:21.785 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:00:21.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:21 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:22.471 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade status' 2024-05-20T23:00:22.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:22 smithi055 ceph-mon[86116]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:22.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:22 smithi055 ceph-mon[88498]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:22.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:22 smithi138 ceph-mon[67823]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:24.586 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:00:24.587 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": null, 2024-05-20T23:00:24.587 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": false, 2024-05-20T23:00:24.587 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "", 2024-05-20T23:00:24.587 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:00:24.587 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": null, 2024-05-20T23:00:24.587 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "", 2024-05-20T23:00:24.587 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:00:24.587 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:00:24.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:24 smithi138 ceph-mon[67823]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:24.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:24 smithi055 ceph-mon[86116]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:24.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:24 smithi055 ceph-mon[88498]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:25.249 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T23:00:25.828 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:25 smithi055 ceph-mon[86116]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:25.828 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:25 smithi055 ceph-mon[88498]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:25.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:25 smithi138 ceph-mon[67823]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:26.565 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:26 smithi055 ceph-mon[86116]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:26.565 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:26 smithi055 ceph-mon[88498]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:26.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:26 smithi138 ceph-mon[67823]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:27.361 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T23:00:27.550 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:00:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:00:27] "GET /metrics HTTP/1.1" 200 38100 "" "Prometheus/2.43.0" 2024-05-20T23:00:27.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:27 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/3042276751' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:00:27.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:27 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/3042276751' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:00:27.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:27 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/3042276751' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:00:28.040 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-05-20T23:00:28.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:28 smithi055 ceph-mon[86116]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:28.677 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:28 smithi055 ceph-mon[88498]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:28.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:28 smithi138 ceph-mon[67823]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:30.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:30 smithi138 ceph-mon[67823]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:30.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:30 smithi138 ceph-mon[67823]: from='client.64288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:30 smithi055 ceph-mon[86116]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:30 smithi055 ceph-mon[86116]: from='client.64288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:30.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:30 smithi055 ceph-mon[88498]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:30.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:30 smithi055 ceph-mon[88498]: from='client.64288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:31.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:31 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:31.928 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:31 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:31.928 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:31 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:32.300 INFO:teuthology.orchestra.run.smithi055.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:00:32.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:32 smithi138 ceph-mon[67823]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:32.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:32 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:32.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:32 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:32.859 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:32 smithi055 ceph-mon[86116]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:32.859 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:32 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:32.859 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:32 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:32.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:32 smithi055 ceph-mon[88498]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:32.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:32 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:32.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:32 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:32.890 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-20T23:00:33.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:33 smithi138 ceph-mon[67823]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:00:33.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:33.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:33.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:33.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:33 smithi138 ceph-mon[67823]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:00:33.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[88498]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:00:33.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:33.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:33.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:33.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[88498]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:00:33.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[86116]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:00:33.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:33.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:33.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:33.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:33 smithi055 ceph-mon[86116]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:00:34.964 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:00:35.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:34 smithi055 ceph-mon[88498]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:35.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:35.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:34 smithi055 ceph-mon[86116]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:35.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:35.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:34 smithi138 ceph-mon[67823]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:35.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:35.672 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:00:35.672 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (5m) 36s ago 27m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:00:35.672 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (5m) 3m ago 27m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:00:35.672 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (5m) 36s ago 27m 60.2M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (7m) 3m ago 32m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (17m) 36s ago 34m 506M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (3m) 36s ago 34m 46.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (4m) 3m ago 32m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (3m) 36s ago 33m 37.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (6m) 36s ago 28m 16.0M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (6m) 3m ago 28m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (76s) 36s ago 32m 50.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (41s) 36s ago 31m 12.0M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (31m) 36s ago 31m 55.3M 2993M 17.2.0 e1d6a67b021e 7d48270c74d5 2024-05-20T23:00:35.673 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (31m) 36s ago 31m 55.1M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T23:00:35.674 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (30m) 3m ago 30m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T23:00:35.674 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (30m) 3m ago 30m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T23:00:35.674 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (29m) 3m ago 29m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T23:00:35.674 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (29m) 3m ago 29m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T23:00:35.674 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (6m) 3m ago 28m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:00:35.674 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (27m) 36s ago 27m 77.0M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:00:35.674 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (27m) 3m ago 27m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:00:36.107 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:00:36.107 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:00:36.107 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 7 2024-05-20T23:00:36.108 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:00:36.109 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:00:36.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:35 smithi055 ceph-mon[86116]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:36.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:35 smithi055 ceph-mon[88498]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:36.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:35 smithi138 ceph-mon[67823]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:36.460 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:00:36.460 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T23:00:36.460 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T23:00:36.460 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-05-20T23:00:36.460 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:00:36.460 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "2/8 daemons upgraded", 2024-05-20T23:00:36.461 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T23:00:36.461 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:00:36.461 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:00:37.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='client.54366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mgr 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mon 2024-05-20T23:00:37.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all crash 2024-05-20T23:00:37.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T23:00:37.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T23:00:37.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: Upgrade: osd.2 is safe to restart 2024-05-20T23:00:37.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:37 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/1383380930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.425 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:00:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:00:37] "GET /metrics HTTP/1.1" 200 38099 "" "Prometheus/2.43.0" 2024-05-20T23:00:37.425 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='client.54366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:37.425 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mgr 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mon 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all crash 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: Upgrade: osd.2 is safe to restart 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/1383380930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='client.54366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:00:37.427 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mgr 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mon 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all crash 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: Upgrade: osd.2 is safe to restart 2024-05-20T23:00:37.428 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:37 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/1383380930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:38.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:38 smithi138 ceph-mon[67823]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:38.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:38 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:38.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:38 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T23:00:38.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:38 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:38.392 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:38 smithi055 ceph-mon[86116]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:38.392 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:38 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:38.392 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:38 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T23:00:38.392 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:38 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:38.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:38 smithi055 ceph-mon[88498]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:00:38.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:38 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:38.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:38 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T23:00:38.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:38 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:39 smithi138 ceph-mon[67823]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:39 smithi138 ceph-mon[67823]: Upgrade: Updating osd.2 2024-05-20T23:00:39.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:39 smithi138 ceph-mon[67823]: Deploying daemon osd.2 on smithi055 2024-05-20T23:00:39.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:39 smithi055 ceph-mon[88498]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:39.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:39 smithi055 ceph-mon[88498]: Upgrade: Updating osd.2 2024-05-20T23:00:39.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:39 smithi055 ceph-mon[88498]: Deploying daemon osd.2 on smithi055 2024-05-20T23:00:39.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:39 smithi055 ceph-mon[86116]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:39.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:39 smithi055 ceph-mon[86116]: Upgrade: Updating osd.2 2024-05-20T23:00:39.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:39 smithi055 ceph-mon[86116]: Deploying daemon osd.2 on smithi055 2024-05-20T23:00:41.055 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:41 smithi055 ceph-mon[86116]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:41.056 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:41 smithi055 systemd[1]: Stopping Ceph osd.2 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:00:41.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:41 smithi138 ceph-mon[67823]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:41.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:41 smithi055 ceph-mon[88498]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:00:41.483 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[38186]: 2024-05-20T23:00:41.274+0000 7f542b8e6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:00:41.483 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[38186]: 2024-05-20T23:00:41.274+0000 7f542b8e6700 -1 osd.2 124 *** Got signal Terminated *** 2024-05-20T23:00:41.483 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[38186]: 2024-05-20T23:00:41.274+0000 7f542b8e6700 -1 osd.2 124 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:00:42.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:42 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:42.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:42 smithi138 ceph-mon[67823]: osd.2 marked itself down and dead 2024-05-20T23:00:42.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:42 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:42.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:42 smithi055 ceph-mon[86116]: osd.2 marked itself down and dead 2024-05-20T23:00:42.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:42 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:42.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:42 smithi055 ceph-mon[88498]: osd.2 marked itself down and dead 2024-05-20T23:00:42.884 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:42 smithi055 podman[106642]: 2024-05-20 23:00:42.606947055 +0000 UTC m=+1.443520836 container died 7d48270c74d52b83dd6e6787264b3b308ff2e1a62754fc8f8906b1f06e443f4d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, release=754, distribution-scope=public, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vcs-type=git, version=8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD) 2024-05-20T23:00:43.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:43 smithi055 ceph-mon[86116]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:43.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:43 smithi055 ceph-mon[86116]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:00:43.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:43 smithi055 ceph-mon[86116]: osdmap e125: 8 total, 7 up, 8 in 2024-05-20T23:00:43.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:43 smithi055 ceph-mon[88498]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:43.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:43 smithi055 ceph-mon[88498]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:00:43.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:43 smithi055 ceph-mon[88498]: osdmap e125: 8 total, 7 up, 8 in 2024-05-20T23:00:43.233 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:43 smithi055 podman[106642]: 2024-05-20 23:00:43.016570192 +0000 UTC m=+1.853143970 container cleanup 7d48270c74d52b83dd6e6787264b3b308ff2e1a62754fc8f8906b1f06e443f4d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, release=754, distribution-scope=public, RELEASE=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git) 2024-05-20T23:00:43.233 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:43 smithi055 bash[106642]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2 2024-05-20T23:00:43.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:43 smithi138 ceph-mon[67823]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:00:43.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:43 smithi138 ceph-mon[67823]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:00:43.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:43 smithi138 ceph-mon[67823]: osdmap e125: 8 total, 7 up, 8 in 2024-05-20T23:00:43.586 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:43 smithi055 podman[106662]: 2024-05-20 23:00:43.263893928 +0000 UTC m=+0.645223945 container remove 7d48270c74d52b83dd6e6787264b3b308ff2e1a62754fc8f8906b1f06e443f4d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, maintainer=Guillaume Abrioux , release=754, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, version=8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container) 2024-05-20T23:00:43.880 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:43 smithi055 podman[106751]: 2024-05-20 23:00:43.497762465 +0000 UTC m=+0.018212731 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:00:43.881 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:43 smithi055 podman[106751]: 2024-05-20 23:00:43.611381609 +0000 UTC m=+0.131831898 container create 39a992b2c3764add5d98fc80434be77d5d406660f8fd0fd96cb7c00a53f39d38 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-20T23:00:43.881 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:43 smithi055 podman[106751]: 2024-05-20 23:00:43.797560226 +0000 UTC m=+0.318010495 container init 39a992b2c3764add5d98fc80434be77d5d406660f8fd0fd96cb7c00a53f39d38 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-20T23:00:43.881 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:43 smithi055 podman[106751]: 2024-05-20 23:00:43.801170708 +0000 UTC m=+0.321620961 container start 39a992b2c3764add5d98fc80434be77d5d406660f8fd0fd96cb7c00a53f39d38 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux ) 2024-05-20T23:00:43.881 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:43 smithi055 podman[106751]: 2024-05-20 23:00:43.889302636 +0000 UTC m=+0.409752909 container attach 39a992b2c3764add5d98fc80434be77d5d406660f8fd0fd96cb7c00a53f39d38 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:00:44.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:44 smithi055 ceph-mon[86116]: osdmap e126: 8 total, 7 up, 8 in 2024-05-20T23:00:44.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:44 smithi055 ceph-mon[88498]: osdmap e126: 8 total, 7 up, 8 in 2024-05-20T23:00:44.233 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:44 smithi055 podman[106751]: 2024-05-20 23:00:44.116564361 +0000 UTC m=+0.637014615 container died 39a992b2c3764add5d98fc80434be77d5d406660f8fd0fd96cb7c00a53f39d38 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-05-20T23:00:44.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:44 smithi138 ceph-mon[67823]: osdmap e126: 8 total, 7 up, 8 in 2024-05-20T23:00:44.983 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:44 smithi055 podman[106778]: 2024-05-20 23:00:44.718587482 +0000 UTC m=+0.596177318 container remove 39a992b2c3764add5d98fc80434be77d5d406660f8fd0fd96cb7c00a53f39d38 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0) 2024-05-20T23:00:44.983 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:44 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.2.service: Deactivated successfully. 2024-05-20T23:00:44.983 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:44 smithi055 systemd[1]: Stopped Ceph osd.2 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:00:44.983 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:44 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.2.service: Consumed 14.151s CPU time. 2024-05-20T23:00:44.983 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:44 smithi055 systemd[1]: Starting Ceph osd.2 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:00:45.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:45 smithi138 ceph-mon[67823]: pgmap v117: 161 pgs: 1 active+undersized+degraded, 11 stale+active+clean, 6 active+undersized, 143 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-05-20T23:00:45.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:45 smithi138 ceph-mon[67823]: Health check failed: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded (PG_DEGRADED) 2024-05-20T23:00:45.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:45 smithi055 ceph-mon[86116]: pgmap v117: 161 pgs: 1 active+undersized+degraded, 11 stale+active+clean, 6 active+undersized, 143 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-05-20T23:00:45.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:45 smithi055 ceph-mon[86116]: Health check failed: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded (PG_DEGRADED) 2024-05-20T23:00:45.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:45 smithi055 ceph-mon[88498]: pgmap v117: 161 pgs: 1 active+undersized+degraded, 11 stale+active+clean, 6 active+undersized, 143 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-05-20T23:00:45.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:45 smithi055 ceph-mon[88498]: Health check failed: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded (PG_DEGRADED) 2024-05-20T23:00:45.432 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 podman[106880]: 2024-05-20 23:00:45.083413583 +0000 UTC m=+0.018063590 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:00:45.432 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 podman[106880]: 2024-05-20 23:00:45.203423228 +0000 UTC m=+0.138073223 container create ea418fa4b76eeee1c86b1486e6af5b8d7192fe08bedb4e9f7ccb1c61bbe2a1f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5) 2024-05-20T23:00:45.432 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 podman[106880]: 2024-05-20 23:00:45.440483084 +0000 UTC m=+0.375133101 container init ea418fa4b76eeee1c86b1486e6af5b8d7192fe08bedb4e9f7ccb1c61bbe2a1f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T23:00:45.733 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 podman[106880]: 2024-05-20 23:00:45.444196939 +0000 UTC m=+0.378846941 container start ea418fa4b76eeee1c86b1486e6af5b8d7192fe08bedb4e9f7ccb1c61bbe2a1f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:00:45.733 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 podman[106880]: 2024-05-20 23:00:45.540314974 +0000 UTC m=+0.474964990 container attach ea418fa4b76eeee1c86b1486e6af5b8d7192fe08bedb4e9f7ccb1c61bbe2a1f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate[106895]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate[106895]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 bash[106880]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 bash[106880]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 bash[106880]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 bash[106880]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 bash[106880]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 bash[106880]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 bash[106880]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-20T23:00:46.203 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate[106895]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-20T23:00:46.204 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate[106895]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-20T23:00:46.204 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate[106895]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-20T23:00:46.204 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate[106895]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-20T23:00:46.204 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate[106895]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-20T23:00:46.204 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:45 smithi055 podman[106880]: 2024-05-20 23:00:45.936116464 +0000 UTC m=+0.870766474 container died ea418fa4b76eeee1c86b1486e6af5b8d7192fe08bedb4e9f7ccb1c61bbe2a1f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-05-20T23:00:46.733 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:46 smithi055 podman[107014]: 2024-05-20 23:00:46.461350608 +0000 UTC m=+0.519118991 container remove ea418fa4b76eeee1c86b1486e6af5b8d7192fe08bedb4e9f7ccb1c61bbe2a1f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-05-20T23:00:46.733 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:46 smithi055 podman[107077]: 2024-05-20 23:00:46.670611009 +0000 UTC m=+0.103635669 container create 632f1b435da2e0fec3f451bec9454fba4a31f8b2fbbd9f7383814420d5e55ab5 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:00:46.733 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:46 smithi055 podman[107077]: 2024-05-20 23:00:46.585175873 +0000 UTC m=+0.018200536 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:00:47.107 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:46 smithi055 podman[107077]: 2024-05-20 23:00:46.873862803 +0000 UTC m=+0.306887466 container init 632f1b435da2e0fec3f451bec9454fba4a31f8b2fbbd9f7383814420d5e55ab5 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514) 2024-05-20T23:00:47.107 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:46 smithi055 podman[107077]: 2024-05-20 23:00:46.877317612 +0000 UTC m=+0.310342280 container start 632f1b435da2e0fec3f451bec9454fba4a31f8b2fbbd9f7383814420d5e55ab5 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-20T23:00:47.107 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:46 smithi055 ceph-osd[107096]: -- 172.21.15.55:0/2880744157 <== mon.1 v2:172.21.15.55:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558174483680 con 0x55817444cc00 2024-05-20T23:00:47.107 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:46 smithi055 bash[107077]: 632f1b435da2e0fec3f451bec9454fba4a31f8b2fbbd9f7383814420d5e55ab5 2024-05-20T23:00:47.107 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:46 smithi055 systemd[1]: Started Ceph osd.2 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:00:47.421 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:00:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:00:47] "GET /metrics HTTP/1.1" 200 38099 "" "Prometheus/2.43.0" 2024-05-20T23:00:47.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[86116]: pgmap v118: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:47.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:47.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:47.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:47.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:47.422 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[88498]: pgmap v118: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:47.423 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:47.423 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:47.423 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:47.423 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:47.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:47 smithi138 ceph-mon[67823]: pgmap v118: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:47.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:47.733 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[107091]: 2024-05-20T23:00:47.430+0000 7f55ef9b47c0 -1 Falling back to public interface 2024-05-20T23:00:49.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:49 smithi055 ceph-mon[86116]: pgmap v119: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:49.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:49 smithi055 ceph-mon[88498]: pgmap v119: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:49.408 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:49 smithi138 ceph-mon[67823]: pgmap v119: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:50.464 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[107091]: 2024-05-20T23:00:50.151+0000 7f55ef9b47c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-05-20T23:00:50.465 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[107091]: 2024-05-20T23:00:50.175+0000 7f55ef9b47c0 -1 osd.2 124 log_to_monitors true 2024-05-20T23:00:50.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:50 smithi055 ceph-mon[86116]: pgmap v120: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:50.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:50.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:50.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:50 smithi055 ceph-mon[86116]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T23:00:50.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:50 smithi055 ceph-mon[88498]: pgmap v120: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:50.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:50.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:50.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:50 smithi055 ceph-mon[88498]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T23:00:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:50 smithi138 ceph-mon[67823]: pgmap v120: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:50.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:00:50.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:50 smithi138 ceph-mon[67823]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T23:00:51.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:51 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:51.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:51 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 53/627 objects degraded (8.453%), 11 pgs degraded (PG_DEGRADED) 2024-05-20T23:00:51.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:51 smithi138 ceph-mon[67823]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T23:00:51.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:51 smithi138 ceph-mon[67823]: osdmap e127: 8 total, 7 up, 8 in 2024-05-20T23:00:51.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:51 smithi138 ceph-mon[67823]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:00:51.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:51 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:51.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:51 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:51.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:51 smithi138 ceph-mon[67823]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' 2024-05-20T23:00:51.982 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:00:51 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[107091]: 2024-05-20T23:00:51.517+0000 7f55dce03700 -1 osd.2 124 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T23:00:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 53/627 objects degraded (8.453%), 11 pgs degraded (PG_DEGRADED) 2024-05-20T23:00:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[86116]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T23:00:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[86116]: osdmap e127: 8 total, 7 up, 8 in 2024-05-20T23:00:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[86116]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:00:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[86116]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' 2024-05-20T23:00:51.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:00:51.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 53/627 objects degraded (8.453%), 11 pgs degraded (PG_DEGRADED) 2024-05-20T23:00:51.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[88498]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T23:00:51.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[88498]: osdmap e127: 8 total, 7 up, 8 in 2024-05-20T23:00:51.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[88498]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:00:51.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:51.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:51.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:51 smithi055 ceph-mon[88498]: from='osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936]' entity='osd.2' 2024-05-20T23:00:52.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:52 smithi138 ceph-mon[67823]: pgmap v122: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:52.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:52 smithi138 ceph-mon[67823]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:00:52.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:52 smithi138 ceph-mon[67823]: osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936] boot 2024-05-20T23:00:52.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:52 smithi138 ceph-mon[67823]: osdmap e128: 8 total, 8 up, 8 in 2024-05-20T23:00:52.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T23:00:52.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[86116]: pgmap v122: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[86116]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:00:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[86116]: osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936] boot 2024-05-20T23:00:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[86116]: osdmap e128: 8 total, 8 up, 8 in 2024-05-20T23:00:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T23:00:52.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[88498]: pgmap v122: 161 pgs: 11 active+undersized+degraded, 36 active+undersized, 114 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 53/627 objects degraded (8.453%) 2024-05-20T23:00:52.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[88498]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:00:52.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[88498]: osd.2 [v2:172.21.15.55:6818/1239880936,v1:172.21.15.55:6819/1239880936] boot 2024-05-20T23:00:52.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[88498]: osdmap e128: 8 total, 8 up, 8 in 2024-05-20T23:00:52.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T23:00:53.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:53 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:53.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:53 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:53.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:53 smithi055 ceph-mon[88498]: osdmap e129: 8 total, 8 up, 8 in 2024-05-20T23:00:53.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:53 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:53.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:53 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:53.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:53 smithi055 ceph-mon[86116]: osdmap e129: 8 total, 8 up, 8 in 2024-05-20T23:00:54.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:53 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:54.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:53 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:54.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:53 smithi138 ceph-mon[67823]: osdmap e129: 8 total, 8 up, 8 in 2024-05-20T23:00:54.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:54 smithi055 ceph-mon[86116]: pgmap v124: 161 pgs: 5 peering, 10 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 48/627 objects degraded (7.656%) 2024-05-20T23:00:54.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:54.840 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:54 smithi055 ceph-mon[88498]: pgmap v124: 161 pgs: 5 peering, 10 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 48/627 objects degraded (7.656%) 2024-05-20T23:00:54.840 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:55.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:54 smithi138 ceph-mon[67823]: pgmap v124: 161 pgs: 5 peering, 10 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 48/627 objects degraded (7.656%) 2024-05-20T23:00:55.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:56.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[88498]: pgmap v126: 161 pgs: 5 peering, 1 active+undersized+degraded, 12 active+undersized, 143 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 1/627 objects degraded (0.159%) 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded (PG_DEGRADED) 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[86116]: pgmap v126: 161 pgs: 5 peering, 1 active+undersized+degraded, 12 active+undersized, 143 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 1/627 objects degraded (0.159%) 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded (PG_DEGRADED) 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:56.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:56.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:56.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:56.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:56 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:56 smithi138 ceph-mon[67823]: pgmap v126: 161 pgs: 5 peering, 1 active+undersized+degraded, 12 active+undersized, 143 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 1/627 objects degraded (0.159%) 2024-05-20T23:00:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:56 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded (PG_DEGRADED) 2024-05-20T23:00:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:56 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:56 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:56 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:56 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:57.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:56 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:57.671 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:00:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:00:57] "GET /metrics HTTP/1.1" 200 38101 "" "Prometheus/2.43.0" 2024-05-20T23:00:57.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:57.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:00:57.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: Upgrade: Finalizing container_image settings 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:00:57.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:57.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: Upgrade: Finalizing container_image settings 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:00:57.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:00:57.987 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: Upgrade: Complete! 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:57.988 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: Upgrade: Complete! 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:57.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:00:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:58.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: Upgrade: Finalizing container_image settings 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:00:58.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:58.108 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: Upgrade: Complete! 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:00:58.109 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:00:58.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:58 smithi055 ceph-mon[88498]: pgmap v127: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 805 B/s rd, 0 op/s 2024-05-20T23:00:58.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:58 smithi055 ceph-mon[88498]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded) 2024-05-20T23:00:58.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:00:58 smithi055 ceph-mon[88498]: Cluster is now healthy 2024-05-20T23:00:58.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:58 smithi055 ceph-mon[86116]: pgmap v127: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 805 B/s rd, 0 op/s 2024-05-20T23:00:58.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:58 smithi055 ceph-mon[86116]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded) 2024-05-20T23:00:58.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:00:58 smithi055 ceph-mon[86116]: Cluster is now healthy 2024-05-20T23:00:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:58 smithi138 ceph-mon[67823]: pgmap v127: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 805 B/s rd, 0 op/s 2024-05-20T23:00:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:58 smithi138 ceph-mon[67823]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded) 2024-05-20T23:00:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:00:58 smithi138 ceph-mon[67823]: Cluster is now healthy 2024-05-20T23:01:00.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:00 smithi138 ceph-mon[67823]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-20T23:01:00.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:00 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:00.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:00 smithi055 ceph-mon[88498]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-20T23:01:00.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:00 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:00.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:00 smithi055 ceph-mon[86116]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-20T23:01:00.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:00 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:01.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:01 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:01.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:01 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:01.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:01 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:02.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:02 smithi138 ceph-mon[67823]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T23:01:02.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:02 smithi055 ceph-mon[86116]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T23:01:02.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:02 smithi055 ceph-mon[88498]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T23:01:04.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:04 smithi055 ceph-mon[88498]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:01:04.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:04.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:04.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:04 smithi055 ceph-mon[86116]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:01:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:05.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:04 smithi138 ceph-mon[67823]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:01:05.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:05.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:06.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:06 smithi055 ceph-mon[86116]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-20T23:01:06.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:06 smithi055 ceph-mon[88498]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-20T23:01:07.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:06 smithi138 ceph-mon[67823]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-20T23:01:07.482 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:01:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:01:07] "GET /metrics HTTP/1.1" 200 38109 "" "Prometheus/2.43.0" 2024-05-20T23:01:07.623 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T23:01:07.939 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:07 smithi055 ceph-mon[86116]: from='client.64330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:07.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:07 smithi055 ceph-mon[88498]: from='client.64330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:08.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:07 smithi138 ceph-mon[67823]: from='client.64330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:08.888 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:08 smithi055 ceph-mon[86116]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:08.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:08 smithi055 ceph-mon[88498]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:09.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:08 smithi138 ceph-mon[67823]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:09.592 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:01:09.592 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (6m) 18s ago 28m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:01:09.592 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (5m) 3m ago 28m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:01:09.592 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (5m) 18s ago 27m 63.9M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:01:09.592 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (7m) 3m ago 33m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:01:09.592 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (17m) 18s ago 34m 508M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:01:09.592 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (3m) 18s ago 34m 48.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (4m) 3m ago 33m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (3m) 18s ago 33m 40.8M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (7m) 18s ago 29m 15.6M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (7m) 3m ago 29m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (110s) 18s ago 32m 51.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (75s) 18s ago 32m 51.0M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (23s) 18s ago 32m 12.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (31m) 18s ago 31m 56.6M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (31m) 3m ago 31m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (30m) 3m ago 30m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (30m) 3m ago 30m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (30m) 3m ago 30m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (6m) 3m ago 29m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:01:09.593 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (27m) 18s ago 27m 77.3M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:01:09.594 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (27m) 3m ago 27m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:01:10.344 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-05-20T23:01:10.876 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:10 smithi055 ceph-mon[86116]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:10.876 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:10 smithi055 ceph-mon[86116]: from='client.54402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:10.876 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:10 smithi055 ceph-mon[88498]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:10.876 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:10 smithi055 ceph-mon[88498]: from='client.54402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:11.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:10 smithi138 ceph-mon[67823]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:11.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:10 smithi138 ceph-mon[67823]: from='client.54402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:11.971 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:11 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:11.972 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:11 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:12.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:11 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:12.527 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:01:12.864 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:12 smithi055 ceph-mon[86116]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:12.864 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:12 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2234348044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:12.864 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:12 smithi055 ceph-mon[88498]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:12.865 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:12 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2234348044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:13.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:12 smithi138 ceph-mon[67823]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:13.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:12 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2234348044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:13.293 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-05-20T23:01:14.928 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:14 smithi055 ceph-mon[86116]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:14.928 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:14 smithi055 ceph-mon[88498]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:15.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:14 smithi138 ceph-mon[67823]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:16 smithi055 ceph-mon[86116]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:16.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:16 smithi055 ceph-mon[86116]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:16.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:16 smithi055 ceph-mon[88498]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:16.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:16 smithi055 ceph-mon[88498]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:17.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:16 smithi138 ceph-mon[67823]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:17.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:16 smithi138 ceph-mon[67823]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:17.516 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:01:17.516 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:01:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:01:17] "GET /metrics HTTP/1.1" 200 38109 "" "Prometheus/2.43.0" 2024-05-20T23:01:18.293 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade status' 2024-05-20T23:01:18.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:18 smithi055 ceph-mon[86116]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:18.867 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:18 smithi055 ceph-mon[88498]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:19.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:18 smithi138 ceph-mon[67823]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:19.907 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:19 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:19.908 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:19 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:20.007 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:19 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:20.347 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:01:20.348 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": null, 2024-05-20T23:01:20.348 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": false, 2024-05-20T23:01:20.348 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "", 2024-05-20T23:01:20.348 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:01:20.348 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": null, 2024-05-20T23:01:20.348 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "", 2024-05-20T23:01:20.348 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:01:20.348 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:01:20.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:20 smithi055 ceph-mon[86116]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:20.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:20 smithi055 ceph-mon[88498]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:21.014 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T23:01:21.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:20 smithi138 ceph-mon[67823]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:21.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:21 smithi055 ceph-mon[86116]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:21.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:21 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:21.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:21 smithi055 ceph-mon[88498]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:21.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:21 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:22.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:21 smithi138 ceph-mon[67823]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:22.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:21 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:22.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:22 smithi055 ceph-mon[86116]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:22.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:22 smithi055 ceph-mon[88498]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:23.099 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T23:01:23.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:22 smithi138 ceph-mon[67823]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:23.809 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-05-20T23:01:23.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:23 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/3338423478' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:01:23.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:23 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/3338423478' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:01:24.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:23 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/3338423478' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:01:24.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:24 smithi055 ceph-mon[88498]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:24.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:24 smithi055 ceph-mon[86116]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:25.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:24 smithi138 ceph-mon[67823]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:27.093 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:26 smithi055 ceph-mon[86116]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:27.093 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:26 smithi055 ceph-mon[86116]: from='client.54432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:27.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:26 smithi055 ceph-mon[88498]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:27.094 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:26 smithi055 ceph-mon[88498]: from='client.54432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:27.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:26 smithi138 ceph-mon[67823]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:27.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:26 smithi138 ceph-mon[67823]: from='client.54432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:27.428 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:01:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:01:27] "GET /metrics HTTP/1.1" 200 38110 "" "Prometheus/2.43.0" 2024-05-20T23:01:28.318 INFO:teuthology.orchestra.run.smithi055.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:01:28.962 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[88498]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[86116]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:28.963 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:01:28.964 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:28.991 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-20T23:01:29.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:28 smithi138 ceph-mon[67823]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:29.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:29.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:29.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:29.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:01:29.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:29.819 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:29 smithi055 ceph-mon[86116]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:01:29.819 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:29 smithi055 ceph-mon[86116]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:01:29.820 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:29 smithi055 ceph-mon[88498]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:01:29.820 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:29 smithi055 ceph-mon[88498]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:01:30.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:29 smithi138 ceph-mon[67823]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:01:30.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:29 smithi138 ceph-mon[67823]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:01:30.915 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:30 smithi055 ceph-mon[86116]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:30.915 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:30 smithi055 ceph-mon[88498]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:31.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:30 smithi138 ceph-mon[67823]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:31.279 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:01:31.967 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:01:31.967 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (6m) 40s ago 28m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:01:31.967 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (6m) 4m ago 28m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:01:31.967 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (6m) 40s ago 28m 63.9M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:01:31.967 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (8m) 4m ago 33m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:01:31.967 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (18m) 40s ago 35m 508M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:01:31.967 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (4m) 40s ago 35m 48.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:01:31.967 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (5m) 4m ago 33m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:01:31.967 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (3m) 40s ago 34m 40.8M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (7m) 40s ago 29m 15.6M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (7m) 4m ago 29m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (2m) 40s ago 33m 51.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (97s) 40s ago 32m 51.0M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (45s) 40s ago 32m 12.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (32m) 40s ago 32m 56.6M 2993M 17.2.0 e1d6a67b021e 5b7d415d63ec 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (31m) 4m ago 31m 55.5M 3248M 17.2.0 e1d6a67b021e 49d9d87d5e97 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (31m) 4m ago 31m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (30m) 4m ago 30m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (30m) 4m ago 30m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (7m) 4m ago 29m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (28m) 40s ago 28m 77.3M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:01:31.968 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (28m) 4m ago 28m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:01:32.077 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:31 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:32.077 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:31 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:32.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:31 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:32.393 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:01:32.394 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T23:01:32.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:01:32.395 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:01:32.395 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-05-20T23:01:32.395 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T23:01:32.395 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:01:32.395 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:01:32.749 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:01:32.749 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T23:01:32.749 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T23:01:32.749 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-20T23:01:32.749 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:01:32.749 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "3/8 daemons upgraded", 2024-05-20T23:01:32.749 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T23:01:32.749 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:01:32.749 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:01:33.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:33.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:33.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:33.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:33.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mgr 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mon 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all crash 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T23:01:33.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: Upgrade: osd.3 is safe to restart 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2875766806' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mgr 2024-05-20T23:01:33.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mon 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all crash 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: Upgrade: osd.3 is safe to restart 2024-05-20T23:01:33.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:33 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2875766806' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:33.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mgr 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mon 2024-05-20T23:01:33.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all crash 2024-05-20T23:01:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T23:01:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T23:01:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: Upgrade: osd.3 is safe to restart 2024-05-20T23:01:33.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:33 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2875766806' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:34.448 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:34 smithi055 ceph-mon[86116]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:34.448 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:34.448 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T23:01:34.449 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:34.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:34 smithi055 ceph-mon[88498]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:34.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:34.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T23:01:34.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:34.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:34 smithi138 ceph-mon[67823]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:01:34.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:34.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T23:01:34.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:35.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:35 smithi055 ceph-mon[88498]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:35.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:35 smithi055 ceph-mon[88498]: Upgrade: Updating osd.3 2024-05-20T23:01:35.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:35 smithi055 ceph-mon[88498]: Deploying daemon osd.3 on smithi055 2024-05-20T23:01:35.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:35 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:35.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:35 smithi055 ceph-mon[86116]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:35.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:35 smithi055 ceph-mon[86116]: Upgrade: Updating osd.3 2024-05-20T23:01:35.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:35 smithi055 ceph-mon[86116]: Deploying daemon osd.3 on smithi055 2024-05-20T23:01:35.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:35 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:35.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:35 smithi138 ceph-mon[67823]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:01:35.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:35 smithi138 ceph-mon[67823]: Upgrade: Updating osd.3 2024-05-20T23:01:35.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:35 smithi138 ceph-mon[67823]: Deploying daemon osd.3 on smithi055 2024-05-20T23:01:35.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:35 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:36.983 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:36 smithi055 systemd[1]: Stopping Ceph osd.3 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:01:37.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:37 smithi055 ceph-mon[88498]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:37.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:37 smithi055 ceph-mon[88498]: osd.3 marked itself down and dead 2024-05-20T23:01:37.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:01:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:01:37] "GET /metrics HTTP/1.1" 200 38105 "" "Prometheus/2.43.0" 2024-05-20T23:01:37.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:37 smithi055 ceph-mon[86116]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:37.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:37 smithi055 ceph-mon[86116]: osd.3 marked itself down and dead 2024-05-20T23:01:37.484 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[41150]: 2024-05-20T23:01:37.139+0000 7f2dba685700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:01:37.484 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[41150]: 2024-05-20T23:01:37.139+0000 7f2dba685700 -1 osd.3 129 *** Got signal Terminated *** 2024-05-20T23:01:37.485 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[41150]: 2024-05-20T23:01:37.139+0000 7f2dba685700 -1 osd.3 129 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:01:37.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:37 smithi138 ceph-mon[67823]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:01:37.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:37 smithi138 ceph-mon[67823]: osd.3 marked itself down and dead 2024-05-20T23:01:37.906 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:37 smithi055 podman[113260]: 2024-05-20 23:01:37.640363458 +0000 UTC m=+0.622425063 container died 5b7d415d63ecae2a2766df27fc8ea53d7d1e12de0d3e00afa972e46a3bc9fa07 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3, ceph=True, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.openshift.expose-services=, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc.) 2024-05-20T23:01:38.170 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:38 smithi055 podman[113260]: 2024-05-20 23:01:38.014162457 +0000 UTC m=+0.996224049 container cleanup 5b7d415d63ecae2a2766df27fc8ea53d7d1e12de0d3e00afa972e46a3bc9fa07 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_BRANCH=HEAD, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, release=754, io.openshift.tags=base centos centos-stream, ceph=True, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.expose-services=, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-20T23:01:38.171 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:38 smithi055 bash[113260]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3 2024-05-20T23:01:38.441 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:38 smithi055 ceph-mon[86116]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:01:38.441 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:38 smithi055 ceph-mon[86116]: osdmap e130: 8 total, 7 up, 8 in 2024-05-20T23:01:38.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:38 smithi055 ceph-mon[88498]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:01:38.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:38 smithi055 ceph-mon[88498]: osdmap e130: 8 total, 7 up, 8 in 2024-05-20T23:01:38.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:38 smithi138 ceph-mon[67823]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:01:38.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:38 smithi138 ceph-mon[67823]: osdmap e130: 8 total, 7 up, 8 in 2024-05-20T23:01:38.732 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:38 smithi055 podman[113275]: 2024-05-20 23:01:38.449504591 +0000 UTC m=+0.805821557 container remove 5b7d415d63ecae2a2766df27fc8ea53d7d1e12de0d3e00afa972e46a3bc9fa07 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, maintainer=Guillaume Abrioux , ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_BRANCH=HEAD, distribution-scope=public, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, RELEASE=HEAD, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container) 2024-05-20T23:01:39.086 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:38 smithi055 podman[113367]: 2024-05-20 23:01:38.716383146 +0000 UTC m=+0.018074826 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:01:39.086 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:38 smithi055 podman[113367]: 2024-05-20 23:01:38.825759634 +0000 UTC m=+0.127451294 container create 06a37b0ede45fff38855fe6c56352defaf6201bd462e88371db583f16cc1a0bd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-05-20T23:01:39.086 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:39 smithi055 podman[113367]: 2024-05-20 23:01:39.020104867 +0000 UTC m=+0.321796547 container init 06a37b0ede45fff38855fe6c56352defaf6201bd462e88371db583f16cc1a0bd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-05-20T23:01:39.086 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:39 smithi055 podman[113367]: 2024-05-20 23:01:39.023844937 +0000 UTC m=+0.325536612 container start 06a37b0ede45fff38855fe6c56352defaf6201bd462e88371db583f16cc1a0bd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux ) 2024-05-20T23:01:39.086 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:39 smithi055 podman[113367]: 2024-05-20 23:01:39.0947969 +0000 UTC m=+0.396488582 container attach 06a37b0ede45fff38855fe6c56352defaf6201bd462e88371db583f16cc1a0bd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:01:39.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:39 smithi055 ceph-mon[86116]: pgmap v148: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:01:39.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:39 smithi055 ceph-mon[86116]: osdmap e131: 8 total, 7 up, 8 in 2024-05-20T23:01:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:39 smithi055 ceph-mon[88498]: pgmap v148: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:01:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:39 smithi055 ceph-mon[88498]: osdmap e131: 8 total, 7 up, 8 in 2024-05-20T23:01:39.483 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:39 smithi055 podman[113367]: 2024-05-20 23:01:39.332849271 +0000 UTC m=+0.634540939 container died 06a37b0ede45fff38855fe6c56352defaf6201bd462e88371db583f16cc1a0bd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0) 2024-05-20T23:01:39.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:39 smithi138 ceph-mon[67823]: pgmap v148: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:01:39.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:39 smithi138 ceph-mon[67823]: osdmap e131: 8 total, 7 up, 8 in 2024-05-20T23:01:40.109 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:39 smithi055 podman[113393]: 2024-05-20 23:01:39.873853323 +0000 UTC m=+0.535151701 container remove 06a37b0ede45fff38855fe6c56352defaf6201bd462e88371db583f16cc1a0bd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS) 2024-05-20T23:01:40.110 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:39 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.3.service: Deactivated successfully. 2024-05-20T23:01:40.110 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:39 smithi055 systemd[1]: Stopped Ceph osd.3 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:01:40.110 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:39 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.3.service: Consumed 18.140s CPU time. 2024-05-20T23:01:40.441 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:40 smithi055 ceph-mon[86116]: pgmap v150: 161 pgs: 11 active+undersized+degraded, 16 stale+active+clean, 16 active+undersized, 118 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 41/627 objects degraded (6.539%) 2024-05-20T23:01:40.441 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:40 smithi055 ceph-mon[86116]: Health check failed: Degraded data redundancy: 41/627 objects degraded (6.539%), 11 pgs degraded (PG_DEGRADED) 2024-05-20T23:01:40.442 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:40 smithi055 ceph-mon[88498]: pgmap v150: 161 pgs: 11 active+undersized+degraded, 16 stale+active+clean, 16 active+undersized, 118 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 41/627 objects degraded (6.539%) 2024-05-20T23:01:40.442 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:40 smithi055 ceph-mon[88498]: Health check failed: Degraded data redundancy: 41/627 objects degraded (6.539%), 11 pgs degraded (PG_DEGRADED) 2024-05-20T23:01:40.442 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:40 smithi055 systemd[1]: Starting Ceph osd.3 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:01:40.442 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:40 smithi055 podman[113491]: 2024-05-20 23:01:40.242886401 +0000 UTC m=+0.018600101 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:01:40.442 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:40 smithi055 podman[113491]: 2024-05-20 23:01:40.350323124 +0000 UTC m=+0.126036816 container create 54824c8b7d1499f39e3e00287738b8b6a89da2c92817a908e2c9030f47b98b33 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD) 2024-05-20T23:01:40.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:40 smithi138 ceph-mon[67823]: pgmap v150: 161 pgs: 11 active+undersized+degraded, 16 stale+active+clean, 16 active+undersized, 118 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 41/627 objects degraded (6.539%) 2024-05-20T23:01:40.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:40 smithi138 ceph-mon[67823]: Health check failed: Degraded data redundancy: 41/627 objects degraded (6.539%), 11 pgs degraded (PG_DEGRADED) 2024-05-20T23:01:40.733 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:40 smithi055 podman[113491]: 2024-05-20 23:01:40.553652403 +0000 UTC m=+0.329366095 container init 54824c8b7d1499f39e3e00287738b8b6a89da2c92817a908e2c9030f47b98b33 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0) 2024-05-20T23:01:40.733 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:40 smithi055 podman[113491]: 2024-05-20 23:01:40.557484244 +0000 UTC m=+0.333197948 container start 54824c8b7d1499f39e3e00287738b8b6a89da2c92817a908e2c9030f47b98b33 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:01:40.733 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:40 smithi055 podman[113491]: 2024-05-20 23:01:40.627933451 +0000 UTC m=+0.403647164 container attach 54824c8b7d1499f39e3e00287738b8b6a89da2c92817a908e2c9030f47b98b33 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-20T23:01:41.271 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate[113505]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-20T23:01:41.271 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate[113505]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-20T23:01:41.271 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 bash[113491]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-20T23:01:41.271 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 bash[113491]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-20T23:01:41.271 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 bash[113491]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-20T23:01:41.271 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 bash[113491]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-20T23:01:41.272 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 bash[113491]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-20T23:01:41.272 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 bash[113491]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-20T23:01:41.272 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 bash[113491]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-20T23:01:41.272 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate[113505]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-20T23:01:41.272 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate[113505]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-20T23:01:41.272 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate[113505]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-20T23:01:41.272 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate[113505]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-20T23:01:41.272 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate[113505]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-20T23:01:41.272 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 podman[113491]: 2024-05-20 23:01:41.047149218 +0000 UTC m=+0.822862917 container died 54824c8b7d1499f39e3e00287738b8b6a89da2c92817a908e2c9030f47b98b33 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-05-20T23:01:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:41 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:41.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:41 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:41.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:41 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:41.932 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 podman[113641]: 2024-05-20 23:01:41.640853552 +0000 UTC m=+0.587619783 container remove 54824c8b7d1499f39e3e00287738b8b6a89da2c92817a908e2c9030f47b98b33 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T23:01:41.932 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 podman[113702]: 2024-05-20 23:01:41.842022583 +0000 UTC m=+0.098762669 container create b7c4fb200e8e8a989c46586eb4f1dd06f6f55e09ef9a037acd1dc64c7a73ca42 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-05-20T23:01:41.932 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:41 smithi055 podman[113702]: 2024-05-20 23:01:41.761761534 +0000 UTC m=+0.018501638 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:01:42.232 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:42 smithi055 podman[113702]: 2024-05-20 23:01:42.061275006 +0000 UTC m=+0.318015098 container init b7c4fb200e8e8a989c46586eb4f1dd06f6f55e09ef9a037acd1dc64c7a73ca42 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-05-20T23:01:42.233 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:42 smithi055 podman[113702]: 2024-05-20 23:01:42.064703277 +0000 UTC m=+0.321443364 container start b7c4fb200e8e8a989c46586eb4f1dd06f6f55e09ef9a037acd1dc64c7a73ca42 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, ceph=True) 2024-05-20T23:01:42.233 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:42 smithi055 ceph-osd[113721]: -- 172.21.15.55:0/3048446526 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562ec3fb3680 con 0x562ec3f7d000 2024-05-20T23:01:42.233 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:42 smithi055 bash[113702]: b7c4fb200e8e8a989c46586eb4f1dd06f6f55e09ef9a037acd1dc64c7a73ca42 2024-05-20T23:01:42.233 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:42 smithi055 systemd[1]: Started Ceph osd.3 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:01:42.983 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[113717]: 2024-05-20T23:01:42.610+0000 7feb2396a7c0 -1 Falling back to public interface 2024-05-20T23:01:43.481 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:43 smithi055 ceph-mon[86116]: pgmap v151: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:43.481 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:43 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:43.481 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:43 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:43.481 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:43 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:43.481 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:43 smithi055 ceph-mon[88498]: pgmap v151: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:43.481 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:43 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:43.481 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:43 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:43.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:43 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:43.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:43 smithi138 ceph-mon[67823]: pgmap v151: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:43.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:43 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:43.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:43 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:43.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:43 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:45.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:45 smithi055 ceph-mon[86116]: pgmap v152: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:45.343 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:45 smithi055 ceph-mon[88498]: pgmap v152: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:45.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:45 smithi138 ceph-mon[67823]: pgmap v152: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:45.732 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[113717]: 2024-05-20T23:01:45.349+0000 7feb2396a7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-05-20T23:01:45.733 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:45 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[113717]: 2024-05-20T23:01:45.378+0000 7feb2396a7c0 -1 osd.3 129 log_to_monitors true 2024-05-20T23:01:46.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:46 smithi055 ceph-mon[86116]: from='osd.3 [v2:172.21.15.55:6826/982350899,v1:172.21.15.55:6827/982350899]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T23:01:46.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:46 smithi055 ceph-mon[86116]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T23:01:46.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:46 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 81/627 objects degraded (12.919%), 26 pgs degraded (PG_DEGRADED) 2024-05-20T23:01:46.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:46 smithi055 ceph-mon[88498]: from='osd.3 [v2:172.21.15.55:6826/982350899,v1:172.21.15.55:6827/982350899]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T23:01:46.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:46 smithi055 ceph-mon[88498]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T23:01:46.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:46 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 81/627 objects degraded (12.919%), 26 pgs degraded (PG_DEGRADED) 2024-05-20T23:01:46.483 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:01:46 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[113717]: 2024-05-20T23:01:46.291+0000 7feb14dc1700 -1 osd.3 129 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T23:01:46.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:46 smithi138 ceph-mon[67823]: from='osd.3 [v2:172.21.15.55:6826/982350899,v1:172.21.15.55:6827/982350899]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T23:01:46.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:46 smithi138 ceph-mon[67823]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T23:01:46.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:46 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 81/627 objects degraded (12.919%), 26 pgs degraded (PG_DEGRADED) 2024-05-20T23:01:47.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:47 smithi138 ceph-mon[67823]: pgmap v153: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:47 smithi138 ceph-mon[67823]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T23:01:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:47 smithi138 ceph-mon[67823]: osdmap e132: 8 total, 7 up, 8 in 2024-05-20T23:01:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:47 smithi138 ceph-mon[67823]: from='osd.3 [v2:172.21.15.55:6826/982350899,v1:172.21.15.55:6827/982350899]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:01:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:47 smithi138 ceph-mon[67823]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:01:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:47.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:47.734 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:01:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:01:47] "GET /metrics HTTP/1.1" 200 38105 "" "Prometheus/2.43.0" 2024-05-20T23:01:47.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[86116]: pgmap v153: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:47.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[86116]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T23:01:47.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[86116]: osdmap e132: 8 total, 7 up, 8 in 2024-05-20T23:01:47.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[86116]: from='osd.3 [v2:172.21.15.55:6826/982350899,v1:172.21.15.55:6827/982350899]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:01:47.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[86116]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:01:47.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:47.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:47.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[88498]: pgmap v153: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:47.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[88498]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T23:01:47.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[88498]: osdmap e132: 8 total, 7 up, 8 in 2024-05-20T23:01:47.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[88498]: from='osd.3 [v2:172.21.15.55:6826/982350899,v1:172.21.15.55:6827/982350899]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:01:47.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[88498]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-05-20T23:01:47.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:47.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:48.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:48 smithi138 ceph-mon[67823]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:01:48.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:48 smithi138 ceph-mon[67823]: osd.3 [v2:172.21.15.55:6826/982350899,v1:172.21.15.55:6827/982350899] boot 2024-05-20T23:01:48.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:48 smithi138 ceph-mon[67823]: osdmap e133: 8 total, 8 up, 8 in 2024-05-20T23:01:48.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:48 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T23:01:48.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:48 smithi138 ceph-mon[67823]: pgmap v156: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:48.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:48 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:48.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:48 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:48.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[86116]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:01:48.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[86116]: osd.3 [v2:172.21.15.55:6826/982350899,v1:172.21.15.55:6827/982350899] boot 2024-05-20T23:01:48.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[86116]: osdmap e133: 8 total, 8 up, 8 in 2024-05-20T23:01:48.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T23:01:48.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[86116]: pgmap v156: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:48.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:48.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[88498]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:01:48.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[88498]: osd.3 [v2:172.21.15.55:6826/982350899,v1:172.21.15.55:6827/982350899] boot 2024-05-20T23:01:48.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[88498]: osdmap e133: 8 total, 8 up, 8 in 2024-05-20T23:01:48.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T23:01:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[88498]: pgmap v156: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 81/627 objects degraded (12.919%) 2024-05-20T23:01:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:48.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:48 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:49 smithi055 ceph-mon[86116]: osdmap e134: 8 total, 8 up, 8 in 2024-05-20T23:01:49.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:49 smithi055 ceph-mon[88498]: osdmap e134: 8 total, 8 up, 8 in 2024-05-20T23:01:49.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:49 smithi138 ceph-mon[67823]: osdmap e134: 8 total, 8 up, 8 in 2024-05-20T23:01:50.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:50 smithi055 ceph-mon[86116]: pgmap v158: 161 pgs: 19 active+undersized+degraded, 31 active+undersized, 111 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/627 objects degraded (8.931%) 2024-05-20T23:01:50.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:50.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:50.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:50 smithi055 ceph-mon[88498]: pgmap v158: 161 pgs: 19 active+undersized+degraded, 31 active+undersized, 111 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/627 objects degraded (8.931%) 2024-05-20T23:01:50.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:50.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:50 smithi138 ceph-mon[67823]: pgmap v158: 161 pgs: 19 active+undersized+degraded, 31 active+undersized, 111 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 56/627 objects degraded (8.931%) 2024-05-20T23:01:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:01:51.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:51 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:51.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:51 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 56/627 objects degraded (8.931%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T23:01:51.980 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:51 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:51.980 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:51 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 56/627 objects degraded (8.931%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T23:01:51.981 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:51 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:01:51.981 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:51 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 56/627 objects degraded (8.931%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T23:01:52.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:52.820 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/627 objects degraded (8.931%), 19 pgs degraded) 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: Cluster is now healthy 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T23:01:52.821 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:52 smithi138 ceph-mon[67823]: Upgrade: osd.4 is safe to restart 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/627 objects degraded (8.931%), 19 pgs degraded) 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: Cluster is now healthy 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:52.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[86116]: Upgrade: osd.4 is safe to restart 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/627 objects degraded (8.931%), 19 pgs degraded) 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: Cluster is now healthy 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:52.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T23:01:52.985 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:52 smithi055 ceph-mon[88498]: Upgrade: osd.4 is safe to restart 2024-05-20T23:01:55.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:54 smithi138 ceph-mon[67823]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:55.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:54 smithi138 ceph-mon[67823]: Upgrade: Updating osd.4 2024-05-20T23:01:55.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:55.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T23:01:55.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:55.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:54 smithi138 ceph-mon[67823]: Deploying daemon osd.4 on smithi138 2024-05-20T23:01:55.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[86116]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[86116]: Upgrade: Updating osd.4 2024-05-20T23:01:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T23:01:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[86116]: Deploying daemon osd.4 on smithi138 2024-05-20T23:01:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:55.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[88498]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:55.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[88498]: Upgrade: Updating osd.4 2024-05-20T23:01:55.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:55.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T23:01:55.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:01:55.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[88498]: Deploying daemon osd.4 on smithi138 2024-05-20T23:01:55.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:01:57.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:56 smithi138 ceph-mon[67823]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:57.052 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:57 smithi138 systemd[1]: Stopping Ceph osd.4 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:01:57.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:56 smithi055 ceph-mon[86116]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:57.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:56 smithi055 ceph-mon[88498]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:57.355 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:57 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[29020]: 2024-05-20T23:01:57.305+0000 7fea32040700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:01:57.356 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:57 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[29020]: 2024-05-20T23:01:57.305+0000 7fea32040700 -1 osd.4 134 *** Got signal Terminated *** 2024-05-20T23:01:57.356 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:57 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[29020]: 2024-05-20T23:01:57.305+0000 7fea32040700 -1 osd.4 134 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:01:57.732 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:01:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:01:57] "GET /metrics HTTP/1.1" 200 37968 "" "Prometheus/2.43.0" 2024-05-20T23:01:58.217 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:57 smithi138 ceph-mon[67823]: osd.4 marked itself down and dead 2024-05-20T23:01:58.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:57 smithi055 ceph-mon[86116]: osd.4 marked itself down and dead 2024-05-20T23:01:58.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:57 smithi055 ceph-mon[88498]: osd.4 marked itself down and dead 2024-05-20T23:01:58.473 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:58 smithi138 podman[77878]: 2024-05-20 23:01:58.2261998 +0000 UTC m=+1.017225989 container died 49d9d87d5e97465e5494c0a9a27638a1f39d7b74959a742e00684c066c1ade4c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, name=centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True) 2024-05-20T23:01:58.765 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:58 smithi138 podman[77878]: 2024-05-20 23:01:58.57309516 +0000 UTC m=+1.364121350 container cleanup 49d9d87d5e97465e5494c0a9a27638a1f39d7b74959a742e00684c066c1ade4c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4, RELEASE=HEAD, name=centos-stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T23:01:58.765 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:58 smithi138 bash[77878]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4 2024-05-20T23:01:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:58 smithi138 ceph-mon[67823]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:58 smithi138 ceph-mon[67823]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:01:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:58 smithi138 ceph-mon[67823]: osdmap e135: 8 total, 7 up, 8 in 2024-05-20T23:01:59.105 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:58 smithi138 podman[77894]: 2024-05-20 23:01:58.774321717 +0000 UTC m=+0.544175642 container remove 49d9d87d5e97465e5494c0a9a27638a1f39d7b74959a742e00684c066c1ade4c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , version=8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-20T23:01:59.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:58 smithi055 ceph-mon[88498]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:59.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:58 smithi055 ceph-mon[88498]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:01:59.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:58 smithi055 ceph-mon[88498]: osdmap e135: 8 total, 7 up, 8 in 2024-05-20T23:01:59.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:58 smithi055 ceph-mon[86116]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:01:59.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:58 smithi055 ceph-mon[86116]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:01:59.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:58 smithi055 ceph-mon[86116]: osdmap e135: 8 total, 7 up, 8 in 2024-05-20T23:01:59.444 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:59 smithi138 podman[77986]: 2024-05-20 23:01:59.101612717 +0000 UTC m=+0.018331505 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:01:59.444 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:59 smithi138 podman[77986]: 2024-05-20 23:01:59.234007473 +0000 UTC m=+0.150726258 container create 28034823aaed5b511ad7127e5e758a7cb5924f5147b2374a095f7db1bf6c73e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-05-20T23:01:59.738 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:59 smithi138 podman[77986]: 2024-05-20 23:01:59.453491996 +0000 UTC m=+0.370210780 container init 28034823aaed5b511ad7127e5e758a7cb5924f5147b2374a095f7db1bf6c73e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0) 2024-05-20T23:01:59.738 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:59 smithi138 podman[77986]: 2024-05-20 23:01:59.456891672 +0000 UTC m=+0.373610454 container start 28034823aaed5b511ad7127e5e758a7cb5924f5147b2374a095f7db1bf6c73e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T23:01:59.738 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:59 smithi138 podman[77986]: 2024-05-20 23:01:59.528271987 +0000 UTC m=+0.444990791 container attach 28034823aaed5b511ad7127e5e758a7cb5924f5147b2374a095f7db1bf6c73e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-05-20T23:02:00.081 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:01:59 smithi138 ceph-mon[67823]: osdmap e136: 8 total, 7 up, 8 in 2024-05-20T23:02:00.082 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:01:59 smithi138 podman[77986]: 2024-05-20 23:01:59.747719348 +0000 UTC m=+0.664438130 container died 28034823aaed5b511ad7127e5e758a7cb5924f5147b2374a095f7db1bf6c73e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:02:00.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:01:59 smithi055 ceph-mon[86116]: osdmap e136: 8 total, 7 up, 8 in 2024-05-20T23:02:00.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:01:59 smithi055 ceph-mon[88498]: osdmap e136: 8 total, 7 up, 8 in 2024-05-20T23:02:00.855 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:00 smithi138 podman[78012]: 2024-05-20 23:02:00.465999384 +0000 UTC m=+0.711911677 container remove 28034823aaed5b511ad7127e5e758a7cb5924f5147b2374a095f7db1bf6c73e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5) 2024-05-20T23:02:00.855 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:00 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.4.service: Deactivated successfully. 2024-05-20T23:02:00.855 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:00 smithi138 systemd[1]: Stopped Ceph osd.4 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:02:00.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:00 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.4.service: Consumed 16.746s CPU time. 2024-05-20T23:02:00.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:00 smithi138 systemd[1]: Starting Ceph osd.4 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:02:00.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:00 smithi055 ceph-mon[86116]: pgmap v165: 161 pgs: 20 peering, 18 stale+active+clean, 123 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:02:00.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:00 smithi055 ceph-mon[86116]: Health check failed: Reduced data availability: 2 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-05-20T23:02:00.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:00 smithi055 ceph-mon[88498]: pgmap v165: 161 pgs: 20 peering, 18 stale+active+clean, 123 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:02:00.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:00 smithi055 ceph-mon[88498]: Health check failed: Reduced data availability: 2 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-05-20T23:02:01.210 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:00 smithi138 ceph-mon[67823]: pgmap v165: 161 pgs: 20 peering, 18 stale+active+clean, 123 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:02:01.211 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:00 smithi138 ceph-mon[67823]: Health check failed: Reduced data availability: 2 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-05-20T23:02:01.211 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:00 smithi138 podman[78112]: 2024-05-20 23:02:00.806846418 +0000 UTC m=+0.016789111 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:02:01.211 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:00 smithi138 podman[78112]: 2024-05-20 23:02:00.933843736 +0000 UTC m=+0.143786409 container create 4b04c2416125fda9b767a91960b2328d22c9b13b8872a373ce39edf027205d69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, ceph=True) 2024-05-20T23:02:01.211 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 podman[78112]: 2024-05-20 23:02:01.144494269 +0000 UTC m=+0.354436961 container init 4b04c2416125fda9b767a91960b2328d22c9b13b8872a373ce39edf027205d69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514) 2024-05-20T23:02:01.211 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 podman[78112]: 2024-05-20 23:02:01.14806503 +0000 UTC m=+0.358007710 container start 4b04c2416125fda9b767a91960b2328d22c9b13b8872a373ce39edf027205d69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-05-20T23:02:01.211 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 podman[78112]: 2024-05-20 23:02:01.219598726 +0000 UTC m=+0.429541432 container attach 4b04c2416125fda9b767a91960b2328d22c9b13b8872a373ce39edf027205d69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:02:01.855 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate[78127]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate[78127]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 bash[78112]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 bash[78112]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 bash[78112]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 bash[78112]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 bash[78112]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 bash[78112]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 bash[78112]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate[78127]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate[78127]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate[78127]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-20T23:02:01.856 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate[78127]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-20T23:02:01.857 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate[78127]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-20T23:02:01.857 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:01 smithi138 podman[78112]: 2024-05-20 23:02:01.616433131 +0000 UTC m=+0.826375820 container died 4b04c2416125fda9b767a91960b2328d22c9b13b8872a373ce39edf027205d69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-05-20T23:02:02.198 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:01 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:02.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:01 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:02.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:01 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:02.449 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:02 smithi138 podman[78242]: 2024-05-20 23:02:02.20715743 +0000 UTC m=+0.584956290 container remove 4b04c2416125fda9b767a91960b2328d22c9b13b8872a373ce39edf027205d69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-05-20T23:02:02.449 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:02 smithi138 podman[78302]: 2024-05-20 23:02:02.331215654 +0000 UTC m=+0.017790100 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:02:02.449 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:02 smithi138 podman[78302]: 2024-05-20 23:02:02.458347603 +0000 UTC m=+0.144922030 container create c32ea1e7b5872a5188932294f22b5cd99a0a3e05f4bc4d09ad6b612c0f02cbf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:02:02.700 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:02 smithi138 podman[78302]: 2024-05-20 23:02:02.685913427 +0000 UTC m=+0.372487865 container init c32ea1e7b5872a5188932294f22b5cd99a0a3e05f4bc4d09ad6b612c0f02cbf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:02:02.701 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:02 smithi138 podman[78302]: 2024-05-20 23:02:02.68938467 +0000 UTC m=+0.375959105 container start c32ea1e7b5872a5188932294f22b5cd99a0a3e05f4bc4d09ad6b612c0f02cbf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:02:03.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:02 smithi138 ceph-mon[67823]: pgmap v166: 161 pgs: 17 active+undersized+degraded, 20 peering, 31 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-05-20T23:02:03.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:02 smithi138 ceph-mon[67823]: Health check failed: Degraded data redundancy: 69/627 objects degraded (11.005%), 17 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:03.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:02 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:03.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:02 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:03.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:02 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:03.106 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:02 smithi138 ceph-osd[78321]: -- 172.21.15.138:0/3473359410 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563a0d29f680 con 0x563a0d269000 2024-05-20T23:02:03.106 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:02 smithi138 bash[78302]: c32ea1e7b5872a5188932294f22b5cd99a0a3e05f4bc4d09ad6b612c0f02cbf4 2024-05-20T23:02:03.106 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:02 smithi138 systemd[1]: Started Ceph osd.4 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:02:03.129 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:02:03.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[88498]: pgmap v166: 161 pgs: 17 active+undersized+degraded, 20 peering, 31 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-05-20T23:02:03.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[88498]: Health check failed: Degraded data redundancy: 69/627 objects degraded (11.005%), 17 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:03.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:03.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:03.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:03.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[86116]: pgmap v166: 161 pgs: 17 active+undersized+degraded, 20 peering, 31 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-05-20T23:02:03.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[86116]: Health check failed: Degraded data redundancy: 69/627 objects degraded (11.005%), 17 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:03.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:03.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:03.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:02 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:03.605 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:03 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:02:03.229+0000 7f6e190d77c0 -1 Falling back to public interface 2024-05-20T23:02:03.809 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:02:03.809 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (7m) 16s ago 29m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (6m) 4m ago 29m 77.2M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (6m) 16s ago 28m 64.0M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (8m) 4m ago 34m 449M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (18m) 16s ago 35m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (4m) 16s ago 35m 50.9M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (5m) 4m ago 34m 37.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (4m) 16s ago 34m 41.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (8m) 16s ago 30m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (8m) 4m ago 30m 15.4M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (2m) 16s ago 33m 52.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (2m) 16s ago 33m 52.5M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:02:03.810 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (77s) 16s ago 32m 49.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:02:03.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (22s) 16s ago 32m 12.7M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:02:03.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 starting - - - 3248M 2024-05-20T23:02:03.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (31m) 4m ago 31m 52.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T23:02:03.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (31m) 4m ago 31m 52.8M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T23:02:03.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (31m) 4m ago 31m 53.4M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T23:02:03.811 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (7m) 4m ago 30m 71.3M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:02:03.811 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (28m) 16s ago 28m 77.6M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:02:03.811 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (28m) 4m ago 28m 79.8M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:02:04.226 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:03 smithi055 ceph-mon[86116]: from='client.64411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:04.226 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:03 smithi055 ceph-mon[88498]: from='client.64411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:04.226 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:02:04.226 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:02:04.226 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:02:04.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 4 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 9 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:02:04.227 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:02:04.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:03 smithi138 ceph-mon[67823]: from='client.64411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:04.578 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:02:04.578 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T23:02:04.578 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T23:02:04.578 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-20T23:02:04.578 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:02:04.578 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "4/8 daemons upgraded", 2024-05-20T23:02:04.579 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T23:02:04.579 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:02:04.579 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:02:05.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[88498]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[88498]: pgmap v167: 161 pgs: 24 active+undersized+degraded, 11 peering, 33 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 96/627 objects degraded (15.311%) 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[88498]: from='client.54489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/1397501462' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[86116]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[86116]: pgmap v167: 161 pgs: 24 active+undersized+degraded, 11 peering, 33 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 96/627 objects degraded (15.311%) 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[86116]: from='client.54489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/1397501462' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:05.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:05.278 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:04 smithi138 ceph-mon[67823]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:05.278 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:04 smithi138 ceph-mon[67823]: pgmap v167: 161 pgs: 24 active+undersized+degraded, 11 peering, 33 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 96/627 objects degraded (15.311%) 2024-05-20T23:02:05.278 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:04 smithi138 ceph-mon[67823]: from='client.54489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:05.278 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:04 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/1397501462' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:05.278 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:05.279 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:06.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:05 smithi055 ceph-mon[86116]: from='client.54501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:06.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:05 smithi055 ceph-mon[86116]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-05-20T23:02:06.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:05 smithi055 ceph-mon[88498]: from='client.54501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:06.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:05 smithi055 ceph-mon[88498]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-05-20T23:02:06.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:05 smithi138 ceph-mon[67823]: from='client.54501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:06.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:05 smithi138 ceph-mon[67823]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-05-20T23:02:06.355 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:02:05.958+0000 7f6e190d77c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-05-20T23:02:06.356 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:05 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:02:05.994+0000 7f6e190d77c0 -1 osd.4 134 log_to_monitors true 2024-05-20T23:02:07.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:06 smithi055 ceph-mon[88498]: pgmap v168: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 109/627 objects degraded (17.384%) 2024-05-20T23:02:07.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:06 smithi055 ceph-mon[88498]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-05-20T23:02:07.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:06 smithi055 ceph-mon[88498]: from='osd.4 [v2:172.21.15.138:6800/3775535417,v1:172.21.15.138:6801/3775535417]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T23:02:07.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:06 smithi055 ceph-mon[88498]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T23:02:07.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:06 smithi055 ceph-mon[86116]: pgmap v168: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 109/627 objects degraded (17.384%) 2024-05-20T23:02:07.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:06 smithi055 ceph-mon[86116]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-05-20T23:02:07.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:06 smithi055 ceph-mon[86116]: from='osd.4 [v2:172.21.15.138:6800/3775535417,v1:172.21.15.138:6801/3775535417]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T23:02:07.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:06 smithi055 ceph-mon[86116]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T23:02:07.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:06 smithi138 ceph-mon[67823]: pgmap v168: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 109/627 objects degraded (17.384%) 2024-05-20T23:02:07.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:06 smithi138 ceph-mon[67823]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-05-20T23:02:07.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:06 smithi138 ceph-mon[67823]: from='osd.4 [v2:172.21.15.138:6800/3775535417,v1:172.21.15.138:6801/3775535417]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T23:02:07.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:06 smithi138 ceph-mon[67823]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T23:02:07.329 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:02:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:02:07.025+0000 7f6e0a52e700 -1 osd.4 134 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T23:02:07.732 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:02:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:02:07] "GET /metrics HTTP/1.1" 200 38192 "" "Prometheus/2.43.0" 2024-05-20T23:02:08.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:08 smithi138 ceph-mon[67823]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T23:02:08.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:08 smithi138 ceph-mon[67823]: from='osd.4 [v2:172.21.15.138:6800/3775535417,v1:172.21.15.138:6801/3775535417]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:08.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:08 smithi138 ceph-mon[67823]: osdmap e137: 8 total, 7 up, 8 in 2024-05-20T23:02:08.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:08 smithi138 ceph-mon[67823]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:08.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:08.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:08 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:08.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[86116]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T23:02:08.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[86116]: from='osd.4 [v2:172.21.15.138:6800/3775535417,v1:172.21.15.138:6801/3775535417]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:08.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[86116]: osdmap e137: 8 total, 7 up, 8 in 2024-05-20T23:02:08.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[86116]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:08.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:08.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[88498]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T23:02:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[88498]: from='osd.4 [v2:172.21.15.138:6800/3775535417,v1:172.21.15.138:6801/3775535417]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:08.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[88498]: osdmap e137: 8 total, 7 up, 8 in 2024-05-20T23:02:08.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[88498]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:08.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:08.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:08 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:09.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:09 smithi138 ceph-mon[67823]: pgmap v170: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 237 B/s rd, 0 op/s; 109/627 objects degraded (17.384%) 2024-05-20T23:02:09.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:09 smithi138 ceph-mon[67823]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:02:09.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:09 smithi138 ceph-mon[67823]: osd.4 [v2:172.21.15.138:6800/3775535417,v1:172.21.15.138:6801/3775535417] boot 2024-05-20T23:02:09.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:09 smithi138 ceph-mon[67823]: osdmap e138: 8 total, 8 up, 8 in 2024-05-20T23:02:09.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:09 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T23:02:09.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:09 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:09.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:09 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:09.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:09 smithi138 ceph-mon[67823]: osdmap e139: 8 total, 8 up, 8 in 2024-05-20T23:02:09.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[86116]: pgmap v170: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 237 B/s rd, 0 op/s; 109/627 objects degraded (17.384%) 2024-05-20T23:02:09.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[86116]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:02:09.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[86116]: osd.4 [v2:172.21.15.138:6800/3775535417,v1:172.21.15.138:6801/3775535417] boot 2024-05-20T23:02:09.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[86116]: osdmap e138: 8 total, 8 up, 8 in 2024-05-20T23:02:09.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T23:02:09.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:09.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:09.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[86116]: osdmap e139: 8 total, 8 up, 8 in 2024-05-20T23:02:09.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[88498]: pgmap v170: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 237 B/s rd, 0 op/s; 109/627 objects degraded (17.384%) 2024-05-20T23:02:09.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[88498]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:02:09.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[88498]: osd.4 [v2:172.21.15.138:6800/3775535417,v1:172.21.15.138:6801/3775535417] boot 2024-05-20T23:02:09.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[88498]: osdmap e138: 8 total, 8 up, 8 in 2024-05-20T23:02:09.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T23:02:09.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:09.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:09.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:09 smithi055 ceph-mon[88498]: osdmap e139: 8 total, 8 up, 8 in 2024-05-20T23:02:10.214 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:10 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 21 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:10.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:10 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 21 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:10.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:10 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 21 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:11.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:11 smithi138 ceph-mon[67823]: pgmap v173: 161 pgs: 15 peering, 21 active+undersized+degraded, 32 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 83/627 objects degraded (13.238%) 2024-05-20T23:02:11.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:11 smithi055 ceph-mon[86116]: pgmap v173: 161 pgs: 15 peering, 21 active+undersized+degraded, 32 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 83/627 objects degraded (13.238%) 2024-05-20T23:02:11.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:11 smithi055 ceph-mon[88498]: pgmap v173: 161 pgs: 15 peering, 21 active+undersized+degraded, 32 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 83/627 objects degraded (13.238%) 2024-05-20T23:02:12.232 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:02:11 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T23:02:11.856+0000 7f06747a2700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-20T23:02:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:02:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:02:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:12.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:12 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:12.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:12.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:12 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:13.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:13 smithi138 ceph-mon[67823]: pgmap v174: 161 pgs: 15 peering, 10 active+undersized+degraded, 13 active+undersized, 123 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-05-20T23:02:13.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:13 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:13.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:13 smithi138 ceph-mon[67823]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-20T23:02:13.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:13 smithi055 ceph-mon[86116]: pgmap v174: 161 pgs: 15 peering, 10 active+undersized+degraded, 13 active+undersized, 123 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-05-20T23:02:13.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:13 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:13.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:13 smithi055 ceph-mon[86116]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-20T23:02:13.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:13 smithi055 ceph-mon[88498]: pgmap v174: 161 pgs: 15 peering, 10 active+undersized+degraded, 13 active+undersized, 123 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 28/627 objects degraded (4.466%) 2024-05-20T23:02:13.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:13 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:13.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:13 smithi055 ceph-mon[88498]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-20T23:02:14.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:14 smithi055 ceph-mon[88498]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 10 pgs degraded) 2024-05-20T23:02:14.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:14 smithi055 ceph-mon[88498]: Cluster is now healthy 2024-05-20T23:02:14.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:14 smithi055 ceph-mon[86116]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 10 pgs degraded) 2024-05-20T23:02:14.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:14 smithi055 ceph-mon[86116]: Cluster is now healthy 2024-05-20T23:02:14.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:14 smithi138 ceph-mon[67823]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 10 pgs degraded) 2024-05-20T23:02:14.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:14 smithi138 ceph-mon[67823]: Cluster is now healthy 2024-05-20T23:02:15.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:15 smithi055 ceph-mon[86116]: pgmap v175: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 784 B/s rd, 0 op/s 2024-05-20T23:02:15.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:15 smithi055 ceph-mon[88498]: pgmap v175: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 784 B/s rd, 0 op/s 2024-05-20T23:02:15.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:15 smithi138 ceph-mon[67823]: pgmap v175: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 784 B/s rd, 0 op/s 2024-05-20T23:02:17.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:17 smithi055 ceph-mon[86116]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-20T23:02:17.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:17 smithi055 ceph-mon[88498]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-20T23:02:17.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:02:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:02:17] "GET /metrics HTTP/1.1" 200 38192 "" "Prometheus/2.43.0" 2024-05-20T23:02:17.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:17 smithi138 ceph-mon[67823]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-20T23:02:19.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:19 smithi055 ceph-mon[86116]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T23:02:19.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:19 smithi055 ceph-mon[88498]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T23:02:19.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:19 smithi138 ceph-mon[67823]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T23:02:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:20 smithi138 ceph-mon[67823]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-05-20T23:02:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:20.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:20.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:20 smithi055 ceph-mon[86116]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-05-20T23:02:20.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:20.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:20.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:20 smithi055 ceph-mon[88498]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 973 B/s rd, 0 op/s 2024-05-20T23:02:20.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:20.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:22.024 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:21 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:22.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:21 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:22.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:21 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:23.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:22 smithi138 ceph-mon[67823]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:23.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:22 smithi055 ceph-mon[86116]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:23.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:22 smithi055 ceph-mon[88498]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:25.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:24 smithi138 ceph-mon[67823]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:02:25.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:24 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:25.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:24 smithi055 ceph-mon[86116]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:02:25.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:24 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:25.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:24 smithi055 ceph-mon[88498]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:02:25.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:24 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:27.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:26 smithi055 ceph-mon[86116]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:02:27.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:26 smithi055 ceph-mon[88498]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:02:27.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:26 smithi138 ceph-mon[67823]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:02:27.297 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:02:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:02:27] "GET /metrics HTTP/1.1" 200 38187 "" "Prometheus/2.43.0" 2024-05-20T23:02:28.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:27 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:28.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:27 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:28.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:27 smithi138 ceph-mon[67823]: Upgrade: osd.5 is safe to restart 2024-05-20T23:02:28.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:27 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:28.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:27 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:28.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:27 smithi055 ceph-mon[86116]: Upgrade: osd.5 is safe to restart 2024-05-20T23:02:28.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:27 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:28.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:27 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T23:02:28.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:27 smithi055 ceph-mon[88498]: Upgrade: osd.5 is safe to restart 2024-05-20T23:02:29.049 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:28 smithi138 ceph-mon[67823]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:29.049 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:29.049 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T23:02:29.049 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:28 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:02:29.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:28 smithi055 ceph-mon[86116]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:29.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:29.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T23:02:29.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:28 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:02:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:28 smithi055 ceph-mon[88498]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T23:02:29.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:28 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:02:30.016 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:29 smithi138 ceph-mon[67823]: Upgrade: Updating osd.5 2024-05-20T23:02:30.016 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:29 smithi138 ceph-mon[67823]: Deploying daemon osd.5 on smithi138 2024-05-20T23:02:30.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:29 smithi055 ceph-mon[86116]: Upgrade: Updating osd.5 2024-05-20T23:02:30.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:29 smithi055 ceph-mon[86116]: Deploying daemon osd.5 on smithi138 2024-05-20T23:02:30.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:29 smithi055 ceph-mon[88498]: Upgrade: Updating osd.5 2024-05-20T23:02:30.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:29 smithi055 ceph-mon[88498]: Deploying daemon osd.5 on smithi138 2024-05-20T23:02:30.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:30 smithi055 ceph-mon[86116]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:02:30.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:30 smithi055 ceph-mon[88498]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:02:31.007 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:30 smithi138 ceph-mon[67823]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:02:31.912 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:31 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:31.913 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:31 smithi138 systemd[1]: Stopping Ceph osd.5 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:02:32.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:31 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:32.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:31 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:32.355 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[31956]: 2024-05-20T23:02:31.919+0000 7f44c8284700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:02:32.355 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[31956]: 2024-05-20T23:02:31.919+0000 7f44c8284700 -1 osd.5 139 *** Got signal Terminated *** 2024-05-20T23:02:32.355 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[31956]: 2024-05-20T23:02:31.919+0000 7f44c8284700 -1 osd.5 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:02:33.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:32 smithi138 ceph-mon[67823]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:33.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:32 smithi138 ceph-mon[67823]: osd.5 marked itself down and dead 2024-05-20T23:02:33.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:32 smithi055 ceph-mon[86116]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:33.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:32 smithi055 ceph-mon[86116]: osd.5 marked itself down and dead 2024-05-20T23:02:33.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:32 smithi055 ceph-mon[88498]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:33.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:32 smithi055 ceph-mon[88498]: osd.5 marked itself down and dead 2024-05-20T23:02:33.554 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:33 smithi138 podman[82348]: 2024-05-20 23:02:33.295860594 +0000 UTC m=+1.481427164 container died de159ce83ebb087d0aa59b4138da9970e235f90887f582a98b9e156c8bdee31b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5, ceph=True, vcs-type=git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, version=8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, name=centos-stream, vendor=Red Hat, Inc.) 2024-05-20T23:02:33.819 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:33 smithi138 podman[82348]: 2024-05-20 23:02:33.653906752 +0000 UTC m=+1.839473316 container cleanup de159ce83ebb087d0aa59b4138da9970e235f90887f582a98b9e156c8bdee31b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5, RELEASE=HEAD, GIT_CLEAN=True, version=8, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, release=754, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-20T23:02:33.820 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:33 smithi138 bash[82348]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5 2024-05-20T23:02:34.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:33 smithi138 ceph-mon[67823]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:02:34.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:33 smithi138 ceph-mon[67823]: osdmap e140: 8 total, 7 up, 8 in 2024-05-20T23:02:34.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:33 smithi138 podman[82366]: 2024-05-20 23:02:33.930351133 +0000 UTC m=+0.630706001 container remove de159ce83ebb087d0aa59b4138da9970e235f90887f582a98b9e156c8bdee31b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, name=centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, version=8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-20T23:02:34.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:33 smithi055 ceph-mon[86116]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:02:34.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:33 smithi055 ceph-mon[86116]: osdmap e140: 8 total, 7 up, 8 in 2024-05-20T23:02:34.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:33 smithi055 ceph-mon[88498]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:02:34.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:33 smithi055 ceph-mon[88498]: osdmap e140: 8 total, 7 up, 8 in 2024-05-20T23:02:34.533 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:34 smithi138 podman[82457]: 2024-05-20 23:02:34.231438867 +0000 UTC m=+0.097495746 container create 8557f964ebc2e18e1bc69b36e9cf1c2cf013952342cd4aa2d1f2c5915d6b31f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:02:34.534 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:34 smithi138 podman[82457]: 2024-05-20 23:02:34.151128005 +0000 UTC m=+0.017184874 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:02:34.534 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:34 smithi138 podman[82457]: 2024-05-20 23:02:34.467065492 +0000 UTC m=+0.333122348 container init 8557f964ebc2e18e1bc69b36e9cf1c2cf013952342cd4aa2d1f2c5915d6b31f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514) 2024-05-20T23:02:34.534 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:34 smithi138 podman[82457]: 2024-05-20 23:02:34.470473926 +0000 UTC m=+0.336530783 container start 8557f964ebc2e18e1bc69b36e9cf1c2cf013952342cd4aa2d1f2c5915d6b31f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-05-20T23:02:34.824 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:34 smithi138 podman[82457]: 2024-05-20 23:02:34.542173319 +0000 UTC m=+0.408230202 container attach 8557f964ebc2e18e1bc69b36e9cf1c2cf013952342cd4aa2d1f2c5915d6b31f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-20T23:02:34.824 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:34 smithi138 podman[82457]: 2024-05-20 23:02:34.763567435 +0000 UTC m=+0.629624303 container died 8557f964ebc2e18e1bc69b36e9cf1c2cf013952342cd4aa2d1f2c5915d6b31f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5) 2024-05-20T23:02:34.954 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:02:35.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:34 smithi138 ceph-mon[67823]: pgmap v186: 161 pgs: 6 peering, 20 stale+active+clean, 135 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:02:35.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:34 smithi138 ceph-mon[67823]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-05-20T23:02:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:34 smithi138 ceph-mon[67823]: osdmap e141: 8 total, 7 up, 8 in 2024-05-20T23:02:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:35.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[86116]: pgmap v186: 161 pgs: 6 peering, 20 stale+active+clean, 135 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:02:35.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[86116]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-05-20T23:02:35.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[86116]: osdmap e141: 8 total, 7 up, 8 in 2024-05-20T23:02:35.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:35.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[88498]: pgmap v186: 161 pgs: 6 peering, 20 stale+active+clean, 135 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:02:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[88498]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-05-20T23:02:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[88498]: osdmap e141: 8 total, 7 up, 8 in 2024-05-20T23:02:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:35.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:35.605 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:35 smithi138 podman[82483]: 2024-05-20 23:02:35.321116007 +0000 UTC m=+0.550687606 container remove 8557f964ebc2e18e1bc69b36e9cf1c2cf013952342cd4aa2d1f2c5915d6b31f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240514, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-05-20T23:02:35.606 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:35 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.5.service: Deactivated successfully. 2024-05-20T23:02:35.606 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:35 smithi138 systemd[1]: Stopped Ceph osd.5 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:02:35.606 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:35 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.5.service: Consumed 16.172s CPU time. 2024-05-20T23:02:35.606 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:35 smithi138 systemd[1]: Starting Ceph osd.5 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:02:35.634 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:02:35.634 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (7m) 48s ago 29m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:02:35.634 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (7m) 28s ago 29m 80.0M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (7m) 48s ago 29m 64.0M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (9m) 28s ago 34m 454M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (19m) 48s ago 36m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (5m) 48s ago 36m 50.9M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (6m) 28s ago 34m 47.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (5m) 48s ago 35m 41.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (8m) 48s ago 30m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (8m) 28s ago 30m 15.6M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (3m) 48s ago 34m 52.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (2m) 48s ago 33m 52.5M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (109s) 48s ago 33m 49.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:02:35.635 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (53s) 48s ago 33m 12.7M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:02:35.636 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (33s) 28s ago 32m 12.2M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 c32ea1e7b587 2024-05-20T23:02:35.636 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (32m) 28s ago 32m 56.5M 3248M 17.2.0 e1d6a67b021e de159ce83ebb 2024-05-20T23:02:35.636 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (31m) 28s ago 31m 56.0M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T23:02:35.636 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (31m) 28s ago 31m 58.5M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T23:02:35.636 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (8m) 28s ago 30m 73.7M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:02:35.636 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (29m) 48s ago 29m 77.6M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:02:35.636 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (29m) 28s ago 29m 81.2M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:02:35.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:35 smithi055 ceph-mon[86116]: from='client.54510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:35.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:35 smithi055 ceph-mon[88498]: from='client.54510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:36.040 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:35 smithi138 ceph-mon[67823]: from='client.54510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:36.041 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:35 smithi138 podman[82584]: 2024-05-20 23:02:35.653441091 +0000 UTC m=+0.017308737 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:02:36.041 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:35 smithi138 podman[82584]: 2024-05-20 23:02:35.755793548 +0000 UTC m=+0.119661195 container create a923cf72dd021b80693e3585eda783281fd604f302e5bc0163cf5af42bda77ba (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, io.buildah.version=1.33.5) 2024-05-20T23:02:36.041 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:35 smithi138 podman[82584]: 2024-05-20 23:02:35.975148132 +0000 UTC m=+0.339015757 container init a923cf72dd021b80693e3585eda783281fd604f302e5bc0163cf5af42bda77ba (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-05-20T23:02:36.041 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:35 smithi138 podman[82584]: 2024-05-20 23:02:35.978596239 +0000 UTC m=+0.342463865 container start a923cf72dd021b80693e3585eda783281fd604f302e5bc0163cf5af42bda77ba (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-20T23:02:36.041 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 podman[82584]: 2024-05-20 23:02:36.049903299 +0000 UTC m=+0.413770945 container attach a923cf72dd021b80693e3585eda783281fd604f302e5bc0163cf5af42bda77ba (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:02:36.057 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:02:36.057 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:02:36.057 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:02:36.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:02:36.057 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:02:36.057 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:02:36.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:02:36.057 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 5 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 10 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:02:36.058 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:02:36.407 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:02:36.408 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T23:02:36.408 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T23:02:36.408 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-20T23:02:36.408 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:02:36.408 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "5/8 daemons upgraded", 2024-05-20T23:02:36.408 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T23:02:36.408 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:02:36.408 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:02:36.795 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate[82599]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-20T23:02:36.795 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate[82599]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-20T23:02:36.795 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 bash[82584]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-20T23:02:36.795 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 bash[82584]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-20T23:02:36.795 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 bash[82584]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-20T23:02:36.795 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 bash[82584]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-20T23:02:36.796 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 bash[82584]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-20T23:02:36.796 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 bash[82584]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-20T23:02:36.796 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 bash[82584]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-20T23:02:36.796 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate[82599]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-20T23:02:36.796 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate[82599]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-20T23:02:36.796 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate[82599]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-20T23:02:36.796 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate[82599]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-20T23:02:36.796 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate[82599]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-20T23:02:36.797 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:36 smithi138 podman[82584]: 2024-05-20 23:02:36.448295909 +0000 UTC m=+0.812163535 container died a923cf72dd021b80693e3585eda783281fd604f302e5bc0163cf5af42bda77ba (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-05-20T23:02:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:36 smithi138 ceph-mon[67823]: from='client.64450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:36 smithi138 ceph-mon[67823]: pgmap v188: 161 pgs: 6 active+undersized+degraded, 18 peering, 10 stale+active+clean, 9 active+undersized, 118 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 17/627 objects degraded (2.711%) 2024-05-20T23:02:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:36 smithi138 ceph-mon[67823]: from='client.54522 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:36 smithi138 ceph-mon[67823]: Health check failed: Degraded data redundancy: 17/627 objects degraded (2.711%), 6 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:37.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:36 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/4193211790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:37.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[86116]: from='client.64450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:37.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[86116]: pgmap v188: 161 pgs: 6 active+undersized+degraded, 18 peering, 10 stale+active+clean, 9 active+undersized, 118 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 17/627 objects degraded (2.711%) 2024-05-20T23:02:37.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[86116]: from='client.54522 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:37.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[86116]: Health check failed: Degraded data redundancy: 17/627 objects degraded (2.711%), 6 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:37.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/4193211790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[88498]: from='client.64450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[88498]: pgmap v188: 161 pgs: 6 active+undersized+degraded, 18 peering, 10 stale+active+clean, 9 active+undersized, 118 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 17/627 objects degraded (2.711%) 2024-05-20T23:02:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[88498]: from='client.54522 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[88498]: Health check failed: Degraded data redundancy: 17/627 objects degraded (2.711%), 6 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:37.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:36 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/4193211790' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:37.455 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:37 smithi138 podman[82716]: 2024-05-20 23:02:37.129354612 +0000 UTC m=+0.672513163 container remove a923cf72dd021b80693e3585eda783281fd604f302e5bc0163cf5af42bda77ba (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-05-20T23:02:37.455 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:37 smithi138 podman[82773]: 2024-05-20 23:02:37.338497818 +0000 UTC m=+0.108628081 container create b35ee64d41acae8baac9cc241f1c7cd95ad689e3d1182c0c2fe81c033fe08ead (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD) 2024-05-20T23:02:37.455 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:37 smithi138 podman[82773]: 2024-05-20 23:02:37.247080964 +0000 UTC m=+0.017211187 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:02:37.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:02:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:02:37] "GET /metrics HTTP/1.1" 200 38192 "" "Prometheus/2.43.0" 2024-05-20T23:02:37.855 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:37 smithi138 podman[82773]: 2024-05-20 23:02:37.574052579 +0000 UTC m=+0.344182797 container init b35ee64d41acae8baac9cc241f1c7cd95ad689e3d1182c0c2fe81c033fe08ead (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514) 2024-05-20T23:02:37.855 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:37 smithi138 podman[82773]: 2024-05-20 23:02:37.577944804 +0000 UTC m=+0.348075028 container start b35ee64d41acae8baac9cc241f1c7cd95ad689e3d1182c0c2fe81c033fe08ead (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:02:37.855 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:37 smithi138 ceph-osd[82792]: -- 172.21.15.138:0/3434468497 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5589e9241680 con 0x5589e920b400 2024-05-20T23:02:37.855 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:37 smithi138 bash[82773]: b35ee64d41acae8baac9cc241f1c7cd95ad689e3d1182c0c2fe81c033fe08ead 2024-05-20T23:02:37.855 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:37 smithi138 systemd[1]: Started Ceph osd.5 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:02:38.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:37 smithi138 ceph-mon[67823]: from='client.64468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:38.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:38.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:38.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:37 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:38.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:37 smithi055 ceph-mon[86116]: from='client.64468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:38.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:38.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:38.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:37 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:38.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:37 smithi055 ceph-mon[88498]: from='client.64468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:02:38.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:38.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:38.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:37 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:38.561 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:02:38.115+0000 7f46e83c27c0 -1 Falling back to public interface 2024-05-20T23:02:39.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:38 smithi055 ceph-mon[86116]: pgmap v189: 161 pgs: 15 active+undersized+degraded, 18 peering, 26 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 51/627 objects degraded (8.134%) 2024-05-20T23:02:39.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:38 smithi055 ceph-mon[88498]: pgmap v189: 161 pgs: 15 active+undersized+degraded, 18 peering, 26 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 51/627 objects degraded (8.134%) 2024-05-20T23:02:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:38 smithi138 ceph-mon[67823]: pgmap v189: 161 pgs: 15 active+undersized+degraded, 18 peering, 26 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 51/627 objects degraded (8.134%) 2024-05-20T23:02:40.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:39 smithi055 ceph-mon[86116]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-05-20T23:02:40.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:39 smithi055 ceph-mon[88498]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-05-20T23:02:40.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:39 smithi138 ceph-mon[67823]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-05-20T23:02:40.923 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:40 smithi055 ceph-mon[88498]: pgmap v190: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-20T23:02:40.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:40 smithi055 ceph-mon[86116]: pgmap v190: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-20T23:02:40.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:40 smithi055 ceph-mon[86116]: from='osd.5 [v2:172.21.15.138:6808/332197154,v1:172.21.15.138:6809/332197154]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T23:02:40.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:40 smithi055 ceph-mon[86116]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T23:02:40.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:40 smithi055 ceph-mon[88498]: from='osd.5 [v2:172.21.15.138:6808/332197154,v1:172.21.15.138:6809/332197154]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T23:02:40.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:40 smithi055 ceph-mon[88498]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T23:02:41.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:40 smithi138 ceph-mon[67823]: pgmap v190: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-20T23:02:41.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:40 smithi138 ceph-mon[67823]: from='osd.5 [v2:172.21.15.138:6808/332197154,v1:172.21.15.138:6809/332197154]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T23:02:41.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:40 smithi138 ceph-mon[67823]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T23:02:41.106 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:40 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:02:40.844+0000 7f46e83c27c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-05-20T23:02:41.106 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:40 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:02:40.874+0000 7f46e83c27c0 -1 osd.5 139 log_to_monitors true 2024-05-20T23:02:41.605 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:02:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:02:41.391+0000 7f46d5811700 -1 osd.5 139 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T23:02:42.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:41 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:42.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:41 smithi138 ceph-mon[67823]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T23:02:42.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:41 smithi138 ceph-mon[67823]: from='osd.5 [v2:172.21.15.138:6808/332197154,v1:172.21.15.138:6809/332197154]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:42.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:41 smithi138 ceph-mon[67823]: osdmap e142: 8 total, 7 up, 8 in 2024-05-20T23:02:42.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:41 smithi138 ceph-mon[67823]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:42.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:41 smithi138 ceph-mon[67823]: from='osd.5 ' entity='osd.5' 2024-05-20T23:02:42.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:42.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[86116]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T23:02:42.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[86116]: from='osd.5 [v2:172.21.15.138:6808/332197154,v1:172.21.15.138:6809/332197154]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:42.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[86116]: osdmap e142: 8 total, 7 up, 8 in 2024-05-20T23:02:42.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[86116]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:42.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[86116]: from='osd.5 ' entity='osd.5' 2024-05-20T23:02:42.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:42.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[88498]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T23:02:42.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[88498]: from='osd.5 [v2:172.21.15.138:6808/332197154,v1:172.21.15.138:6809/332197154]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:42.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[88498]: osdmap e142: 8 total, 7 up, 8 in 2024-05-20T23:02:42.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[88498]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:02:42.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:41 smithi055 ceph-mon[88498]: from='osd.5 ' entity='osd.5' 2024-05-20T23:02:43.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[86116]: pgmap v192: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[86116]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[86116]: osd.5 [v2:172.21.15.138:6808/332197154,v1:172.21.15.138:6809/332197154] boot 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[86116]: osdmap e143: 8 total, 8 up, 8 in 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[88498]: pgmap v192: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:43.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[88498]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:02:43.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[88498]: osd.5 [v2:172.21.15.138:6808/332197154,v1:172.21.15.138:6809/332197154] boot 2024-05-20T23:02:43.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[88498]: osdmap e143: 8 total, 8 up, 8 in 2024-05-20T23:02:43.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:42 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T23:02:43.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:42 smithi138 ceph-mon[67823]: pgmap v192: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-20T23:02:43.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:42 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:43.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:42 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:43.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:42 smithi138 ceph-mon[67823]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:02:43.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:42 smithi138 ceph-mon[67823]: osd.5 [v2:172.21.15.138:6808/332197154,v1:172.21.15.138:6809/332197154] boot 2024-05-20T23:02:43.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:42 smithi138 ceph-mon[67823]: osdmap e143: 8 total, 8 up, 8 in 2024-05-20T23:02:43.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:42 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T23:02:44.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:44 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:44.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:44 smithi055 ceph-mon[86116]: osdmap e144: 8 total, 8 up, 8 in 2024-05-20T23:02:44.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:44 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:44.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:44 smithi055 ceph-mon[86116]: pgmap v195: 161 pgs: 5 activating, 17 active+undersized+degraded, 37 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 62/627 objects degraded (9.888%) 2024-05-20T23:02:44.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:44 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:44.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:44 smithi055 ceph-mon[88498]: osdmap e144: 8 total, 8 up, 8 in 2024-05-20T23:02:44.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:44 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:44.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:44 smithi055 ceph-mon[88498]: pgmap v195: 161 pgs: 5 activating, 17 active+undersized+degraded, 37 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 62/627 objects degraded (9.888%) 2024-05-20T23:02:44.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:44 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:44.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:44 smithi138 ceph-mon[67823]: osdmap e144: 8 total, 8 up, 8 in 2024-05-20T23:02:44.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:44 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:44.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:44 smithi138 ceph-mon[67823]: pgmap v195: 161 pgs: 5 activating, 17 active+undersized+degraded, 37 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 62/627 objects degraded (9.888%) 2024-05-20T23:02:45.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:45 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 62/627 objects degraded (9.888%), 17 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:45.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:45 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 62/627 objects degraded (9.888%), 17 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:45.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:45 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 62/627 objects degraded (9.888%), 17 pgs degraded (PG_DEGRADED) 2024-05-20T23:02:46.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:46 smithi055 ceph-mon[86116]: pgmap v196: 161 pgs: 5 activating, 12 active+undersized+degraded, 25 active+undersized, 119 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 40/627 objects degraded (6.380%) 2024-05-20T23:02:46.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:46 smithi055 ceph-mon[88498]: pgmap v196: 161 pgs: 5 activating, 12 active+undersized+degraded, 25 active+undersized, 119 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 40/627 objects degraded (6.380%) 2024-05-20T23:02:46.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:46 smithi138 ceph-mon[67823]: pgmap v196: 161 pgs: 5 activating, 12 active+undersized+degraded, 25 active+undersized, 119 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 40/627 objects degraded (6.380%) 2024-05-20T23:02:47.482 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:02:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T23:02:47.030+0000 7f06747a2700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-05-20T23:02:47.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:02:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:02:47] "GET /metrics HTTP/1.1" 200 38192 "" "Prometheus/2.43.0" 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:02:48.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[88498]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:48.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:02:48.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:02:48.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:47 smithi055 ceph-mon[86116]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-05-20T23:02:48.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:48.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:02:48.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:47 smithi138 ceph-mon[67823]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-05-20T23:02:49.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:48 smithi055 ceph-mon[86116]: pgmap v197: 161 pgs: 5 activating, 156 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:49.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:48 smithi055 ceph-mon[86116]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/627 objects degraded (6.380%), 12 pgs degraded) 2024-05-20T23:02:49.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:48 smithi055 ceph-mon[86116]: Cluster is now healthy 2024-05-20T23:02:49.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:48 smithi055 ceph-mon[88498]: pgmap v197: 161 pgs: 5 activating, 156 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:49.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:48 smithi055 ceph-mon[88498]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/627 objects degraded (6.380%), 12 pgs degraded) 2024-05-20T23:02:49.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:48 smithi055 ceph-mon[88498]: Cluster is now healthy 2024-05-20T23:02:49.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:48 smithi138 ceph-mon[67823]: pgmap v197: 161 pgs: 5 activating, 156 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:02:49.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:48 smithi138 ceph-mon[67823]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/627 objects degraded (6.380%), 12 pgs degraded) 2024-05-20T23:02:49.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:48 smithi138 ceph-mon[67823]: Cluster is now healthy 2024-05-20T23:02:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:50.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:50 smithi138 ceph-mon[67823]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-05-20T23:02:50.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:50 smithi055 ceph-mon[86116]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-05-20T23:02:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:02:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:50 smithi055 ceph-mon[88498]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-05-20T23:02:51.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:51 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:51.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:51 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:51.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:51 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:02:52.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:52 smithi138 ceph-mon[67823]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 897 B/s rd, 0 op/s 2024-05-20T23:02:52.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:52 smithi055 ceph-mon[86116]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 897 B/s rd, 0 op/s 2024-05-20T23:02:52.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:52 smithi055 ceph-mon[88498]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 897 B/s rd, 0 op/s 2024-05-20T23:02:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:54 smithi138 ceph-mon[67823]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 0 op/s 2024-05-20T23:02:54.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:54 smithi055 ceph-mon[86116]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 0 op/s 2024-05-20T23:02:54.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:54 smithi055 ceph-mon[88498]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 0 op/s 2024-05-20T23:02:55.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:55 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:55.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:55 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:56.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:55 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:02:56.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:56 smithi055 ceph-mon[86116]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T23:02:56.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:56 smithi055 ceph-mon[88498]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T23:02:57.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:56 smithi138 ceph-mon[67823]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-05-20T23:02:57.732 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:02:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:02:57] "GET /metrics HTTP/1.1" 200 38205 "" "Prometheus/2.43.0" 2024-05-20T23:02:58.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:02:58 smithi055 ceph-mon[86116]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T23:02:58.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:02:58 smithi055 ceph-mon[88498]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T23:02:59.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:02:58 smithi138 ceph-mon[67823]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-20T23:03:00.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:00 smithi055 ceph-mon[86116]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:00.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:00 smithi055 ceph-mon[88498]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:01.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:00 smithi138 ceph-mon[67823]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:01.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:01 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:01.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:01 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:01.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:01 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:02.917 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:02 smithi138 ceph-mon[67823]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:03:02.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:02 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:03:02.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:02 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:03:02.918 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:02 smithi138 ceph-mon[67823]: Upgrade: osd.6 is safe to restart 2024-05-20T23:03:02.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:02 smithi055 ceph-mon[88498]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:03:02.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:02 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:03:02.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:02 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:03:02.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:02 smithi055 ceph-mon[88498]: Upgrade: osd.6 is safe to restart 2024-05-20T23:03:02.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:02 smithi055 ceph-mon[86116]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:03:02.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:02 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:03:02.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:02 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T23:03:02.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:02 smithi055 ceph-mon[86116]: Upgrade: osd.6 is safe to restart 2024-05-20T23:03:04.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:04 smithi138 ceph-mon[67823]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:04.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:04 smithi138 ceph-mon[67823]: Upgrade: Updating osd.6 2024-05-20T23:03:04.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:04.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T23:03:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:04 smithi138 ceph-mon[67823]: Deploying daemon osd.6 on smithi138 2024-05-20T23:03:04.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:04 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:04.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[86116]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[86116]: Upgrade: Updating osd.6 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[86116]: Deploying daemon osd.6 on smithi138 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[88498]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[88498]: Upgrade: Updating osd.6 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:04.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T23:03:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[88498]: Deploying daemon osd.6 on smithi138 2024-05-20T23:03:04.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:04 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:06.784 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:03:06.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:06 smithi138 ceph-mon[67823]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:06.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:06 smithi055 ceph-mon[86116]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:06.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:06 smithi055 ceph-mon[88498]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:07.169 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:06 smithi138 systemd[1]: Stopping Ceph osd.6 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:03:07.473 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:03:07.473 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (8m) 80s ago 30m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (7m) 25s ago 30m 80.8M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (7m) 80s ago 29m 64.0M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (9m) 25s ago 35m 454M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (19m) 80s ago 36m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (5m) 80s ago 36m 50.9M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (6m) 25s ago 35m 48.6M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (5m) 80s ago 35m 41.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (9m) 80s ago 31m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (9m) 25s ago 31m 15.2M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (3m) 80s ago 34m 52.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:03:07.474 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (3m) 80s ago 34m 52.5M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:03:07.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (2m) 80s ago 34m 49.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:03:07.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (85s) 80s ago 33m 12.7M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:03:07.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (65s) 25s ago 33m 44.4M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 c32ea1e7b587 2024-05-20T23:03:07.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (30s) 25s ago 32m 12.5M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 b35ee64d41ac 2024-05-20T23:03:07.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (32m) 25s ago 32m 56.7M 3248M 17.2.0 e1d6a67b021e 5756841cf6db 2024-05-20T23:03:07.475 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (32m) 25s ago 32m 59.2M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T23:03:07.475 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (8m) 25s ago 31m 76.9M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:03:07.475 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (29m) 80s ago 29m 77.6M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:03:07.475 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (29m) 25s ago 29m 81.4M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:03:07.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[34833]: 2024-05-20T23:03:07.175+0000 7f2b7e6e3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:03:07.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[34833]: 2024-05-20T23:03:07.175+0000 7f2b7e6e3700 -1 osd.6 144 *** Got signal Terminated *** 2024-05-20T23:03:07.606 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:07 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[34833]: 2024-05-20T23:03:07.175+0000 7f2b7e6e3700 -1 osd.6 144 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:03:07.727 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:03:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:03:07] "GET /metrics HTTP/1.1" 200 38198 "" "Prometheus/2.43.0" 2024-05-20T23:03:07.894 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 6 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:03:07.895 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:03:07.896 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T23:03:07.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:03:07.896 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:03:07.896 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-20T23:03:07.896 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 11 2024-05-20T23:03:07.896 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:03:07.896 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:03:07.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:07 smithi055 ceph-mon[88498]: from='client.54546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:07.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:07 smithi055 ceph-mon[88498]: from='client.54552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:07.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:07 smithi055 ceph-mon[88498]: osd.6 marked itself down and dead 2024-05-20T23:03:07.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:07 smithi055 ceph-mon[86116]: from='client.54546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:07.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:07 smithi055 ceph-mon[86116]: from='client.54552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:07.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:07 smithi055 ceph-mon[86116]: osd.6 marked itself down and dead 2024-05-20T23:03:08.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:07 smithi138 ceph-mon[67823]: from='client.54546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:08.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:07 smithi138 ceph-mon[67823]: from='client.54552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:08.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:07 smithi138 ceph-mon[67823]: osd.6 marked itself down and dead 2024-05-20T23:03:08.241 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:03:08.241 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T23:03:08.241 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T23:03:08.241 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-20T23:03:08.241 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:03:08.241 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "6/8 daemons upgraded", 2024-05-20T23:03:08.241 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T23:03:08.241 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:03:08.242 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:03:08.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:08 smithi138 podman[86976]: 2024-05-20 23:03:08.183926264 +0000 UTC m=+1.132266006 container died 5756841cf6db7b669e24d77ede8fe5e480ecdb74b177416a5255ed8ce40579e9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.expose-services=, name=centos-stream, vcs-type=git) 2024-05-20T23:03:08.729 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:08 smithi138 podman[86976]: 2024-05-20 23:03:08.617699451 +0000 UTC m=+1.566039191 container cleanup 5756841cf6db7b669e24d77ede8fe5e480ecdb74b177416a5255ed8ce40579e9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-type=git, version=8, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-20T23:03:08.729 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:08 smithi138 bash[86976]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6 2024-05-20T23:03:08.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[86116]: from='client.64489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:08.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[86116]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[86116]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[86116]: osdmap e145: 8 total, 7 up, 8 in 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/3897860969' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[86116]: from='client.54570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[88498]: from='client.64489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[88498]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[88498]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[88498]: osdmap e145: 8 total, 7 up, 8 in 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/3897860969' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:08.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:08 smithi055 ceph-mon[88498]: from='client.54570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:09.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:08 smithi138 ceph-mon[67823]: from='client.64489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:09.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:08 smithi138 ceph-mon[67823]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:03:09.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:08 smithi138 ceph-mon[67823]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:03:09.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:08 smithi138 ceph-mon[67823]: osdmap e145: 8 total, 7 up, 8 in 2024-05-20T23:03:09.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:08 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/3897860969' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:09.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:08 smithi138 ceph-mon[67823]: from='client.54570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:09.106 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:08 smithi138 podman[87004]: 2024-05-20 23:03:08.824476393 +0000 UTC m=+0.640738676 container remove 5756841cf6db7b669e24d77ede8fe5e480ecdb74b177416a5255ed8ce40579e9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6, release=754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.expose-services=, maintainer=Guillaume Abrioux , architecture=x86_64, io.openshift.tags=base centos centos-stream) 2024-05-20T23:03:09.530 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:09 smithi138 podman[87093]: 2024-05-20 23:03:09.137862307 +0000 UTC m=+0.017852380 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:03:09.530 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:09 smithi138 podman[87093]: 2024-05-20 23:03:09.245411994 +0000 UTC m=+0.125402049 container create b73fbc266e42f59ba086899e1717db5853d87ad0959c7156bca9728c242b08cd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T23:03:09.829 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:09 smithi138 ceph-mon[67823]: osdmap e146: 8 total, 7 up, 8 in 2024-05-20T23:03:09.829 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:09 smithi138 podman[87093]: 2024-05-20 23:03:09.53962008 +0000 UTC m=+0.419610142 container init b73fbc266e42f59ba086899e1717db5853d87ad0959c7156bca9728c242b08cd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:03:09.829 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:09 smithi138 podman[87093]: 2024-05-20 23:03:09.543122334 +0000 UTC m=+0.423112397 container start b73fbc266e42f59ba086899e1717db5853d87ad0959c7156bca9728c242b08cd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-05-20T23:03:09.829 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:09 smithi138 podman[87093]: 2024-05-20 23:03:09.648009818 +0000 UTC m=+0.527999900 container attach b73fbc266e42f59ba086899e1717db5853d87ad0959c7156bca9728c242b08cd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-05-20T23:03:10.105 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:09 smithi138 podman[87093]: 2024-05-20 23:03:09.838563296 +0000 UTC m=+0.718553365 container died b73fbc266e42f59ba086899e1717db5853d87ad0959c7156bca9728c242b08cd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514) 2024-05-20T23:03:10.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:09 smithi055 ceph-mon[86116]: osdmap e146: 8 total, 7 up, 8 in 2024-05-20T23:03:10.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:09 smithi055 ceph-mon[88498]: osdmap e146: 8 total, 7 up, 8 in 2024-05-20T23:03:10.769 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:10 smithi138 podman[87119]: 2024-05-20 23:03:10.477325417 +0000 UTC m=+0.632414959 container remove b73fbc266e42f59ba086899e1717db5853d87ad0959c7156bca9728c242b08cd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:03:10.769 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:10 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.6.service: Deactivated successfully. 2024-05-20T23:03:10.769 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:10 smithi138 systemd[1]: Stopped Ceph osd.6 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:03:10.769 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:10 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.6.service: Consumed 14.346s CPU time. 2024-05-20T23:03:10.769 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:10 smithi138 systemd[1]: Starting Ceph osd.6 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:03:11.036 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:10 smithi138 podman[87219]: 2024-05-20 23:03:10.810432331 +0000 UTC m=+0.017171266 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:03:11.036 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:10 smithi138 ceph-mon[67823]: pgmap v210: 161 pgs: 1 active+undersized+degraded, 3 peering, 10 stale+active+clean, 4 active+undersized, 143 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 9/627 objects degraded (1.435%) 2024-05-20T23:03:11.036 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:10 smithi138 ceph-mon[67823]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-20T23:03:11.036 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:10 smithi138 ceph-mon[67823]: Health check failed: Degraded data redundancy: 9/627 objects degraded (1.435%), 1 pg degraded (PG_DEGRADED) 2024-05-20T23:03:11.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:10 smithi055 ceph-mon[86116]: pgmap v210: 161 pgs: 1 active+undersized+degraded, 3 peering, 10 stale+active+clean, 4 active+undersized, 143 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 9/627 objects degraded (1.435%) 2024-05-20T23:03:11.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:10 smithi055 ceph-mon[86116]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-20T23:03:11.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:10 smithi055 ceph-mon[86116]: Health check failed: Degraded data redundancy: 9/627 objects degraded (1.435%), 1 pg degraded (PG_DEGRADED) 2024-05-20T23:03:11.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:10 smithi055 ceph-mon[88498]: pgmap v210: 161 pgs: 1 active+undersized+degraded, 3 peering, 10 stale+active+clean, 4 active+undersized, 143 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 9/627 objects degraded (1.435%) 2024-05-20T23:03:11.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:10 smithi055 ceph-mon[88498]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-20T23:03:11.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:10 smithi055 ceph-mon[88498]: Health check failed: Degraded data redundancy: 9/627 objects degraded (1.435%), 1 pg degraded (PG_DEGRADED) 2024-05-20T23:03:11.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 podman[87219]: 2024-05-20 23:03:11.045426165 +0000 UTC m=+0.252165088 container create e8672b1bc61b89bd3fa86a136884db7ab5848858d18a8b429416e5822b4e9dae (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:03:11.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 podman[87219]: 2024-05-20 23:03:11.281137915 +0000 UTC m=+0.487876844 container init e8672b1bc61b89bd3fa86a136884db7ab5848858d18a8b429416e5822b4e9dae (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-05-20T23:03:11.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 podman[87219]: 2024-05-20 23:03:11.284989816 +0000 UTC m=+0.491728757 container start e8672b1bc61b89bd3fa86a136884db7ab5848858d18a8b429416e5822b4e9dae (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:03:11.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 podman[87219]: 2024-05-20 23:03:11.364475193 +0000 UTC m=+0.571214140 container attach e8672b1bc61b89bd3fa86a136884db7ab5848858d18a8b429416e5822b4e9dae (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5) 2024-05-20T23:03:12.101 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:11 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:12.101 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate[87233]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-20T23:03:12.101 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate[87233]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-20T23:03:12.102 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 bash[87219]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-20T23:03:12.102 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 bash[87219]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-20T23:03:12.102 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 bash[87219]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-20T23:03:12.102 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 bash[87219]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-20T23:03:12.102 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 bash[87219]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-20T23:03:12.102 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 bash[87219]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-20T23:03:12.102 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 bash[87219]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-20T23:03:12.102 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate[87233]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-20T23:03:12.103 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate[87233]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-20T23:03:12.103 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate[87233]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-20T23:03:12.103 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate[87233]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-20T23:03:12.103 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate[87233]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-20T23:03:12.103 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:11 smithi138 podman[87219]: 2024-05-20 23:03:11.7567495 +0000 UTC m=+0.963488435 container died e8672b1bc61b89bd3fa86a136884db7ab5848858d18a8b429416e5822b4e9dae (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD) 2024-05-20T23:03:12.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:11 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:12.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:11 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:12.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:12 smithi138 podman[87349]: 2024-05-20 23:03:12.343654889 +0000 UTC m=+0.581339520 container remove e8672b1bc61b89bd3fa86a136884db7ab5848858d18a8b429416e5822b4e9dae (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS) 2024-05-20T23:03:12.813 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:12 smithi138 podman[87408]: 2024-05-20 23:03:12.457878714 +0000 UTC m=+0.017438109 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:03:12.813 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:12 smithi138 podman[87408]: 2024-05-20 23:03:12.569584479 +0000 UTC m=+0.129143916 container create ceee3eb06ecbcf3d7d5280949f569f4420df0bbba1853491d259afd598d249be (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-20T23:03:12.814 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:12 smithi138 ceph-mon[67823]: pgmap v211: 161 pgs: 14 active+undersized+degraded, 3 peering, 6 stale+active+clean, 21 active+undersized, 117 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 50/627 objects degraded (7.974%) 2024-05-20T23:03:13.105 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:12 smithi138 podman[87408]: 2024-05-20 23:03:12.822237519 +0000 UTC m=+0.381796933 container init ceee3eb06ecbcf3d7d5280949f569f4420df0bbba1853491d259afd598d249be (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:03:13.105 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:12 smithi138 podman[87408]: 2024-05-20 23:03:12.826019719 +0000 UTC m=+0.385579116 container start ceee3eb06ecbcf3d7d5280949f569f4420df0bbba1853491d259afd598d249be (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-05-20T23:03:13.105 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:12 smithi138 ceph-osd[87427]: -- 172.21.15.138:0/3380587150 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5632364d9680 con 0x5632364a3400 2024-05-20T23:03:13.105 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:12 smithi138 bash[87408]: ceee3eb06ecbcf3d7d5280949f569f4420df0bbba1853491d259afd598d249be 2024-05-20T23:03:13.106 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:12 smithi138 systemd[1]: Started Ceph osd.6 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:03:13.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:12 smithi055 ceph-mon[86116]: pgmap v211: 161 pgs: 14 active+undersized+degraded, 3 peering, 6 stale+active+clean, 21 active+undersized, 117 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 50/627 objects degraded (7.974%) 2024-05-20T23:03:13.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:12 smithi055 ceph-mon[88498]: pgmap v211: 161 pgs: 14 active+undersized+degraded, 3 peering, 6 stale+active+clean, 21 active+undersized, 117 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 0 op/s; 50/627 objects degraded (7.974%) 2024-05-20T23:03:13.826 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:03:13.369+0000 7f82c7e647c0 -1 Falling back to public interface 2024-05-20T23:03:14.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:13 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:14.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:13 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:14.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:13 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:14.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:13 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:14.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:13 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:14.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:13 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:14.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:13 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:14.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:13 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:14.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:13 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:15.254 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:14 smithi138 ceph-mon[67823]: pgmap v212: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-05-20T23:03:15.255 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:14 smithi138 ceph-mon[67823]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-20T23:03:15.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:14 smithi055 ceph-mon[86116]: pgmap v212: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-05-20T23:03:15.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:14 smithi055 ceph-mon[86116]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-20T23:03:15.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:14 smithi055 ceph-mon[88498]: pgmap v212: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-05-20T23:03:15.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:14 smithi055 ceph-mon[88498]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-20T23:03:16.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:16 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 77/627 objects degraded (12.281%), 20 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:16.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:03:16.093+0000 7f82c7e647c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-05-20T23:03:16.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:03:16.117+0000 7f82c7e647c0 -1 osd.6 144 log_to_monitors true 2024-05-20T23:03:16.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:16 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 77/627 objects degraded (12.281%), 20 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:16.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:16 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 77/627 objects degraded (12.281%), 20 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:17.248 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:17 smithi055 ceph-mon[86116]: pgmap v213: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-05-20T23:03:17.248 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:17 smithi055 ceph-mon[86116]: from='osd.6 [v2:172.21.15.138:6816/226913033,v1:172.21.15.138:6817/226913033]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T23:03:17.248 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:17 smithi055 ceph-mon[86116]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T23:03:17.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:17 smithi055 ceph-mon[88498]: pgmap v213: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-05-20T23:03:17.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:17 smithi055 ceph-mon[88498]: from='osd.6 [v2:172.21.15.138:6816/226913033,v1:172.21.15.138:6817/226913033]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T23:03:17.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:17 smithi055 ceph-mon[88498]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T23:03:17.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:17 smithi138 ceph-mon[67823]: pgmap v213: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-05-20T23:03:17.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:17 smithi138 ceph-mon[67823]: from='osd.6 [v2:172.21.15.138:6816/226913033,v1:172.21.15.138:6817/226913033]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T23:03:17.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:17 smithi138 ceph-mon[67823]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T23:03:17.732 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:03:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:03:17] "GET /metrics HTTP/1.1" 200 38198 "" "Prometheus/2.43.0" 2024-05-20T23:03:17.855 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:03:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:03:17.469+0000 7f82b52b3700 -1 osd.6 144 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T23:03:18.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:18 smithi138 ceph-mon[67823]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T23:03:18.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:18 smithi138 ceph-mon[67823]: from='osd.6 [v2:172.21.15.138:6816/226913033,v1:172.21.15.138:6817/226913033]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:18.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:18 smithi138 ceph-mon[67823]: osdmap e147: 8 total, 7 up, 8 in 2024-05-20T23:03:18.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:18 smithi138 ceph-mon[67823]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:18 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:18 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:18.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:18 smithi138 ceph-mon[67823]: from='osd.6 ' entity='osd.6' 2024-05-20T23:03:18.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[86116]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T23:03:18.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[86116]: from='osd.6 [v2:172.21.15.138:6816/226913033,v1:172.21.15.138:6817/226913033]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:18.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[86116]: osdmap e147: 8 total, 7 up, 8 in 2024-05-20T23:03:18.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[86116]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:18.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:18.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:18.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[86116]: from='osd.6 ' entity='osd.6' 2024-05-20T23:03:18.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[88498]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T23:03:18.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[88498]: from='osd.6 [v2:172.21.15.138:6816/226913033,v1:172.21.15.138:6817/226913033]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:18.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[88498]: osdmap e147: 8 total, 7 up, 8 in 2024-05-20T23:03:18.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[88498]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:18.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:18.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:18.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:18 smithi055 ceph-mon[88498]: from='osd.6 ' entity='osd.6' 2024-05-20T23:03:19.261 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:19 smithi138 ceph-mon[67823]: pgmap v215: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-05-20T23:03:19.261 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:19 smithi138 ceph-mon[67823]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:03:19.261 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:19 smithi138 ceph-mon[67823]: osd.6 [v2:172.21.15.138:6816/226913033,v1:172.21.15.138:6817/226913033] boot 2024-05-20T23:03:19.261 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:19 smithi138 ceph-mon[67823]: osdmap e148: 8 total, 8 up, 8 in 2024-05-20T23:03:19.261 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:19 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T23:03:19.261 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:19 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:19.261 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:19 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:19.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[86116]: pgmap v215: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[86116]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[86116]: osd.6 [v2:172.21.15.138:6816/226913033,v1:172.21.15.138:6817/226913033] boot 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[86116]: osdmap e148: 8 total, 8 up, 8 in 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[88498]: pgmap v215: 161 pgs: 20 active+undersized+degraded, 33 active+undersized, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 77/627 objects degraded (12.281%) 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[88498]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[88498]: osd.6 [v2:172.21.15.138:6816/226913033,v1:172.21.15.138:6817/226913033] boot 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[88498]: osdmap e148: 8 total, 8 up, 8 in 2024-05-20T23:03:19.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T23:03:19.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:19.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:19 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:20.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:20 smithi055 ceph-mon[88498]: osdmap e149: 8 total, 8 up, 8 in 2024-05-20T23:03:20.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:20.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:20.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:20 smithi055 ceph-mon[88498]: pgmap v218: 161 pgs: 4 peering, 19 active+undersized+degraded, 30 active+undersized, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 0 B/s wr, 6 op/s; 68/627 objects degraded (10.845%) 2024-05-20T23:03:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:20 smithi055 ceph-mon[86116]: osdmap e149: 8 total, 8 up, 8 in 2024-05-20T23:03:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:20.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:20 smithi055 ceph-mon[86116]: pgmap v218: 161 pgs: 4 peering, 19 active+undersized+degraded, 30 active+undersized, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 0 B/s wr, 6 op/s; 68/627 objects degraded (10.845%) 2024-05-20T23:03:20.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:20 smithi138 ceph-mon[67823]: osdmap e149: 8 total, 8 up, 8 in 2024-05-20T23:03:20.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:20.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:20.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:20 smithi138 ceph-mon[67823]: pgmap v218: 161 pgs: 4 peering, 19 active+undersized+degraded, 30 active+undersized, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 0 B/s wr, 6 op/s; 68/627 objects degraded (10.845%) 2024-05-20T23:03:21.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:21 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 68/627 objects degraded (10.845%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:21.856 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:21 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:21.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:21 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 68/627 objects degraded (10.845%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:21.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:21 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:21.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:21 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 68/627 objects degraded (10.845%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:21.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:21 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:22.482 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:03:22 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T23:03:22.127+0000 7f06747a2700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-20T23:03:23.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: pgmap v219: 161 pgs: 4 peering, 8 active+undersized+degraded, 15 active+undersized, 134 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 69 op/s; 29/627 objects degraded (4.625%); 0 B/s, 0 objects/s recovering 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:23.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[86116]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: pgmap v219: 161 pgs: 4 peering, 8 active+undersized+degraded, 15 active+undersized, 134 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 69 op/s; 29/627 objects degraded (4.625%); 0 B/s, 0 objects/s recovering 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:23.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:23.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:23.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:23.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:22 smithi055 ceph-mon[88498]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-20T23:03:23.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: pgmap v219: 161 pgs: 4 peering, 8 active+undersized+degraded, 15 active+undersized, 134 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 69 op/s; 29/627 objects degraded (4.625%); 0 B/s, 0 objects/s recovering 2024-05-20T23:03:23.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:23.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:23.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:22 smithi138 ceph-mon[67823]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-05-20T23:03:24.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:24 smithi138 ceph-mon[67823]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded) 2024-05-20T23:03:24.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:24 smithi138 ceph-mon[67823]: Cluster is now healthy 2024-05-20T23:03:24.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:24 smithi055 ceph-mon[86116]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded) 2024-05-20T23:03:24.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:24 smithi055 ceph-mon[86116]: Cluster is now healthy 2024-05-20T23:03:24.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:24 smithi055 ceph-mon[88498]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded) 2024-05-20T23:03:24.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:24 smithi055 ceph-mon[88498]: Cluster is now healthy 2024-05-20T23:03:25.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:25 smithi138 ceph-mon[67823]: pgmap v220: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 98 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:25.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:25 smithi055 ceph-mon[86116]: pgmap v220: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 98 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:25.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:25 smithi055 ceph-mon[88498]: pgmap v220: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 98 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:27.285 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:27 smithi055 ceph-mon[86116]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 0 B/s wr, 107 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:27.285 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:03:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:03:27] "GET /metrics HTTP/1.1" 200 38064 "" "Prometheus/2.43.0" 2024-05-20T23:03:27.286 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:27 smithi055 ceph-mon[88498]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 0 B/s wr, 107 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:27.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:27 smithi138 ceph-mon[67823]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 0 B/s wr, 107 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:29.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:29 smithi138 ceph-mon[67823]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 133 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:29.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:29 smithi055 ceph-mon[86116]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 133 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:29.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:29 smithi055 ceph-mon[88498]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 133 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:31.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:31 smithi138 ceph-mon[67823]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 139 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:31.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:31 smithi055 ceph-mon[86116]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 139 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:31.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:31 smithi055 ceph-mon[88498]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 139 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:32.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:32 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:32.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:32 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:32.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:32 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:33.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:33 smithi138 ceph-mon[67823]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 0 B/s wr, 117 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:33.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:33 smithi055 ceph-mon[88498]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 0 B/s wr, 117 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:33.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:33 smithi055 ceph-mon[86116]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 0 B/s wr, 117 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:35.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:35 smithi138 ceph-mon[67823]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 0 B/s wr, 85 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:35.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:35 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:35.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:35 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:35.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:35 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:35.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:35 smithi055 ceph-mon[86116]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 0 B/s wr, 85 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:35.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:35 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:35.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:35 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:35.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:35 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:35.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:35 smithi055 ceph-mon[88498]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 0 B/s wr, 85 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:35.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:35 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:35.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:35 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:35.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:35 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:37.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:37 smithi138 ceph-mon[67823]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:37.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:37 smithi055 ceph-mon[86116]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:37.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:37 smithi055 ceph-mon[88498]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s; 0 B/s, 0 objects/s recovering 2024-05-20T23:03:37.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:03:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:03:37] "GET /metrics HTTP/1.1" 200 38215 "" "Prometheus/2.43.0" 2024-05-20T23:03:38.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:38 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:38.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:38 smithi138 ceph-mon[67823]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:38.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:38 smithi138 ceph-mon[67823]: Upgrade: osd.7 is safe to restart 2024-05-20T23:03:38.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:38 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:38.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:38 smithi055 ceph-mon[88498]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:38.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:38 smithi055 ceph-mon[88498]: Upgrade: osd.7 is safe to restart 2024-05-20T23:03:38.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:38 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:38.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:38 smithi055 ceph-mon[86116]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T23:03:38.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:38 smithi055 ceph-mon[86116]: Upgrade: osd.7 is safe to restart 2024-05-20T23:03:38.622 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:03:39.323 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:03:39.323 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (8m) 112s ago 30m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:03:39.323 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (8m) 22s ago 30m 81.4M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:03:39.323 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (8m) 112s ago 30m 64.0M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:03:39.323 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (10m) 22s ago 35m 455M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (20m) 112s ago 37m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (6m) 112s ago 37m 50.9M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (7m) 22s ago 35m 48.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (6m) 112s ago 36m 41.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (9m) 112s ago 31m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (9m) 22s ago 31m 15.7M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (4m) 112s ago 35m 52.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (3m) 112s ago 34m 52.5M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (2m) 112s ago 34m 49.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (117s) 112s ago 34m 12.7M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (96s) 22s ago 33m 45.2M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 c32ea1e7b587 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (62s) 22s ago 33m 51.4M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 b35ee64d41ac 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (26s) 22s ago 33m 11.9M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 ceee3eb06ecb 2024-05-20T23:03:39.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (32m) 22s ago 32m 59.5M 3248M 17.2.0 e1d6a67b021e 8a5581a1b902 2024-05-20T23:03:39.325 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (9m) 22s ago 31m 76.9M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:03:39.325 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (30m) 112s ago 30m 77.6M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:03:39.325 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (30m) 22s ago 30m 81.6M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:03:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:39 smithi138 ceph-mon[67823]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 49 op/s 2024-05-20T23:03:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:39 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:39 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T23:03:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:39 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:39.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:39 smithi055 ceph-mon[86116]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 49 op/s 2024-05-20T23:03:39.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:39 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:39.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:39 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T23:03:39.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:39 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:39 smithi055 ceph-mon[88498]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 49 op/s 2024-05-20T23:03:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:39 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:39 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T23:03:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:39 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:39.739 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:03:39.739 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:03:39.739 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:03:39.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:03:39.739 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:03:39.739 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:03:39.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:03:39.739 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 7 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 12 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:03:39.740 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:03:40.085 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:03:40.085 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T23:03:40.085 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T23:03:40.085 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-20T23:03:40.085 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:03:40.086 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "7/8 daemons upgraded", 2024-05-20T23:03:40.086 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T23:03:40.086 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:03:40.086 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:03:40.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:40 smithi138 ceph-mon[67823]: from='client.64510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:40.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:40 smithi138 ceph-mon[67823]: Upgrade: Updating osd.7 2024-05-20T23:03:40.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:40 smithi138 ceph-mon[67823]: Deploying daemon osd.7 on smithi138 2024-05-20T23:03:40.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:40 smithi138 ceph-mon[67823]: from='client.64516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:40.347 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:40 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/1095035409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:40.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[86116]: from='client.64510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:40.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[86116]: Upgrade: Updating osd.7 2024-05-20T23:03:40.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[86116]: Deploying daemon osd.7 on smithi138 2024-05-20T23:03:40.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[86116]: from='client.64516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:40.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/1095035409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:40.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[88498]: from='client.64510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:40.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[88498]: Upgrade: Updating osd.7 2024-05-20T23:03:40.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[88498]: Deploying daemon osd.7 on smithi138 2024-05-20T23:03:40.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[88498]: from='client.64516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:40.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:40 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/1095035409' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:41.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:41 smithi138 ceph-mon[67823]: from='client.54591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:41.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:41 smithi138 ceph-mon[67823]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 0 B/s wr, 18 op/s 2024-05-20T23:03:41.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:41 smithi138 ceph-mon[67823]: from='client.54603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:41.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:41 smithi055 ceph-mon[86116]: from='client.54591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:41.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:41 smithi055 ceph-mon[86116]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 0 B/s wr, 18 op/s 2024-05-20T23:03:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:41 smithi055 ceph-mon[86116]: from='client.54603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:41.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:41 smithi055 ceph-mon[88498]: from='client.54591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:41.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:41 smithi055 ceph-mon[88498]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 0 B/s wr, 18 op/s 2024-05-20T23:03:41.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:41 smithi055 ceph-mon[88498]: from='client.54603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:03:42.396 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:42 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:42.396 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:42 smithi138 systemd[1]: Stopping Ceph osd.7 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:03:42.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:42 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:42.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:42 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:42.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:42 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[37732]: 2024-05-20T23:03:42.404+0000 7eff1ef62700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:03:42.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:42 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[37732]: 2024-05-20T23:03:42.404+0000 7eff1ef62700 -1 osd.7 149 *** Got signal Terminated *** 2024-05-20T23:03:42.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:42 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[37732]: 2024-05-20T23:03:42.404+0000 7eff1ef62700 -1 osd.7 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:03:43.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:43 smithi055 ceph-mon[86116]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:03:43.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:43 smithi055 ceph-mon[86116]: osd.7 marked itself down and dead 2024-05-20T23:03:43.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:43 smithi055 ceph-mon[88498]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:03:43.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:43 smithi055 ceph-mon[88498]: osd.7 marked itself down and dead 2024-05-20T23:03:43.601 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:43 smithi138 ceph-mon[67823]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:03:43.602 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:43 smithi138 ceph-mon[67823]: osd.7 marked itself down and dead 2024-05-20T23:03:43.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:43 smithi138 podman[91477]: 2024-05-20 23:03:43.610774713 +0000 UTC m=+1.312891592 container died 8a5581a1b902f7659869dbe1fcd63ed903b6be3a93a77efe5b15313ececa3375 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7, GIT_BRANCH=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vendor=Red Hat, Inc., architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.buildah.version=1.19.8, RELEASE=HEAD, maintainer=Guillaume Abrioux , distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-20T23:03:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:44 smithi138 ceph-mon[67823]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:03:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:44 smithi138 ceph-mon[67823]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-20T23:03:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:44 smithi138 ceph-mon[67823]: osdmap e150: 8 total, 7 up, 8 in 2024-05-20T23:03:44.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:44 smithi138 ceph-mon[67823]: osdmap e151: 8 total, 7 up, 8 in 2024-05-20T23:03:44.315 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:44 smithi138 podman[91477]: 2024-05-20 23:03:44.071346903 +0000 UTC m=+1.773463781 container cleanup 8a5581a1b902f7659869dbe1fcd63ed903b6be3a93a77efe5b15313ececa3375 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, name=centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-20T23:03:44.315 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:44 smithi138 bash[91477]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7 2024-05-20T23:03:44.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:44 smithi055 ceph-mon[86116]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:03:44.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:44 smithi055 ceph-mon[86116]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-20T23:03:44.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:44 smithi055 ceph-mon[86116]: osdmap e150: 8 total, 7 up, 8 in 2024-05-20T23:03:44.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:44 smithi055 ceph-mon[86116]: osdmap e151: 8 total, 7 up, 8 in 2024-05-20T23:03:44.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:44 smithi055 ceph-mon[88498]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T23:03:44.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:44 smithi055 ceph-mon[88498]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-20T23:03:44.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:44 smithi055 ceph-mon[88498]: osdmap e150: 8 total, 7 up, 8 in 2024-05-20T23:03:44.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:44 smithi055 ceph-mon[88498]: osdmap e151: 8 total, 7 up, 8 in 2024-05-20T23:03:44.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:44 smithi138 podman[91494]: 2024-05-20 23:03:44.323715899 +0000 UTC m=+0.710598467 container remove 8a5581a1b902f7659869dbe1fcd63ed903b6be3a93a77efe5b15313ececa3375 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, ceph=True, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, RELEASE=HEAD, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.expose-services=, release=754, distribution-scope=public, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., architecture=x86_64) 2024-05-20T23:03:44.927 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:44 smithi138 podman[91584]: 2024-05-20 23:03:44.564407097 +0000 UTC m=+0.017428219 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:03:44.927 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:44 smithi138 podman[91584]: 2024-05-20 23:03:44.666566895 +0000 UTC m=+0.119588045 container create 5c22c1fdc13f13280d4bb0a6e2ef00e51e80ef4af7eb5fd47efe3343256e315a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-05-20T23:03:44.927 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:44 smithi138 podman[91584]: 2024-05-20 23:03:44.845325678 +0000 UTC m=+0.298346794 container init 5c22c1fdc13f13280d4bb0a6e2ef00e51e80ef4af7eb5fd47efe3343256e315a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5) 2024-05-20T23:03:44.927 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:44 smithi138 podman[91584]: 2024-05-20 23:03:44.848712867 +0000 UTC m=+0.301733976 container start 5c22c1fdc13f13280d4bb0a6e2ef00e51e80ef4af7eb5fd47efe3343256e315a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True) 2024-05-20T23:03:44.927 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:44 smithi138 podman[91584]: 2024-05-20 23:03:44.936160767 +0000 UTC m=+0.389181894 container attach 5c22c1fdc13f13280d4bb0a6e2ef00e51e80ef4af7eb5fd47efe3343256e315a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-20T23:03:45.187 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:45 smithi138 podman[91584]: 2024-05-20 23:03:45.140774503 +0000 UTC m=+0.593795623 container died 5c22c1fdc13f13280d4bb0a6e2ef00e51e80ef4af7eb5fd47efe3343256e315a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-05-20T23:03:45.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:45 smithi138 ceph-mon[67823]: pgmap v231: 161 pgs: 13 peering, 20 stale+active+clean, 128 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:03:45.440 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:45 smithi138 ceph-mon[67823]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-05-20T23:03:45.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:45 smithi055 ceph-mon[86116]: pgmap v231: 161 pgs: 13 peering, 20 stale+active+clean, 128 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:03:45.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:45 smithi055 ceph-mon[86116]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-05-20T23:03:45.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:45 smithi055 ceph-mon[88498]: pgmap v231: 161 pgs: 13 peering, 20 stale+active+clean, 128 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:03:45.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:45 smithi055 ceph-mon[88498]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-05-20T23:03:46.105 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:45 smithi138 podman[91610]: 2024-05-20 23:03:45.740902321 +0000 UTC m=+0.589639017 container remove 5c22c1fdc13f13280d4bb0a6e2ef00e51e80ef4af7eb5fd47efe3343256e315a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-20T23:03:46.105 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:45 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.7.service: Deactivated successfully. 2024-05-20T23:03:46.105 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:45 smithi138 systemd[1]: Stopped Ceph osd.7 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:03:46.105 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:45 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.7.service: Consumed 16.961s CPU time. 2024-05-20T23:03:46.106 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:45 smithi138 systemd[1]: Starting Ceph osd.7 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:03:46.419 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 podman[91707]: 2024-05-20 23:03:46.075952209 +0000 UTC m=+0.017252701 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:03:46.419 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 podman[91707]: 2024-05-20 23:03:46.217679593 +0000 UTC m=+0.158980083 container create f8fc3b34813391ca4b5c8a75de404e3c4f0779e246a9fe42e97d400e6fb921bb (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-05-20T23:03:46.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 podman[91707]: 2024-05-20 23:03:46.429059607 +0000 UTC m=+0.370360097 container init f8fc3b34813391ca4b5c8a75de404e3c4f0779e246a9fe42e97d400e6fb921bb (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:03:46.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 podman[91707]: 2024-05-20 23:03:46.432933132 +0000 UTC m=+0.374233621 container start f8fc3b34813391ca4b5c8a75de404e3c4f0779e246a9fe42e97d400e6fb921bb (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514) 2024-05-20T23:03:46.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 podman[91707]: 2024-05-20 23:03:46.503918251 +0000 UTC m=+0.445218754 container attach f8fc3b34813391ca4b5c8a75de404e3c4f0779e246a9fe42e97d400e6fb921bb (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T23:03:47.196 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate[91723]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-20T23:03:47.196 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate[91723]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-20T23:03:47.196 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 bash[91707]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-20T23:03:47.196 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 bash[91707]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 bash[91707]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 bash[91707]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 bash[91707]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 bash[91707]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 bash[91707]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate[91723]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate[91723]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate[91723]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate[91723]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate[91723]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-20T23:03:47.197 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:46 smithi138 podman[91707]: 2024-05-20 23:03:46.902921755 +0000 UTC m=+0.844222244 container died f8fc3b34813391ca4b5c8a75de404e3c4f0779e246a9fe42e97d400e6fb921bb (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:03:47.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:47 smithi055 ceph-mon[86116]: pgmap v233: 161 pgs: 17 peering, 19 stale+active+clean, 125 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:03:47.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:47 smithi055 ceph-mon[88498]: pgmap v233: 161 pgs: 17 peering, 19 stale+active+clean, 125 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:03:47.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:03:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:03:47] "GET /metrics HTTP/1.1" 200 38215 "" "Prometheus/2.43.0" 2024-05-20T23:03:47.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:47 smithi138 ceph-mon[67823]: pgmap v233: 161 pgs: 17 peering, 19 stale+active+clean, 125 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:03:47.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:47 smithi138 podman[91848]: 2024-05-20 23:03:47.567391737 +0000 UTC m=+0.659046096 container remove f8fc3b34813391ca4b5c8a75de404e3c4f0779e246a9fe42e97d400e6fb921bb (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-activate, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-05-20T23:03:47.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:47 smithi138 podman[91909]: 2024-05-20 23:03:47.768318203 +0000 UTC m=+0.104470519 container create e41094f18f45c74dcd30f981b34f62923703e0cf01fc70d8e2670a0bc7da8255 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=) 2024-05-20T23:03:47.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:47 smithi138 podman[91909]: 2024-05-20 23:03:47.681538639 +0000 UTC m=+0.017690971 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:03:48.355 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:47 smithi138 podman[91909]: 2024-05-20 23:03:47.970973048 +0000 UTC m=+0.307125386 container init e41094f18f45c74dcd30f981b34f62923703e0cf01fc70d8e2670a0bc7da8255 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-05-20T23:03:48.355 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:47 smithi138 podman[91909]: 2024-05-20 23:03:47.974876912 +0000 UTC m=+0.311029235 container start e41094f18f45c74dcd30f981b34f62923703e0cf01fc70d8e2670a0bc7da8255 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-05-20T23:03:48.355 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:47 smithi138 ceph-osd[91930]: -- 172.21.15.138:0/3127264625 <== mon.2 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eb3d90b680 con 0x55eb3d8d4c00 2024-05-20T23:03:48.355 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:48 smithi138 bash[91909]: e41094f18f45c74dcd30f981b34f62923703e0cf01fc70d8e2670a0bc7da8255 2024-05-20T23:03:48.355 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:48 smithi138 systemd[1]: Started Ceph osd.7 for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:03:48.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:48 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:03:48.518+0000 7fedd61147c0 -1 Falling back to public interface 2024-05-20T23:03:49.240 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:49 smithi138 ceph-mon[67823]: pgmap v234: 161 pgs: 19 active+undersized+degraded, 17 peering, 33 active+undersized, 92 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-05-20T23:03:49.241 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:49.241 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:49.241 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:49.241 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:49 smithi138 ceph-mon[67823]: Health check failed: Degraded data redundancy: 69/627 objects degraded (11.005%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[86116]: pgmap v234: 161 pgs: 19 active+undersized+degraded, 17 peering, 33 active+undersized, 92 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-05-20T23:03:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:49.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[86116]: Health check failed: Degraded data redundancy: 69/627 objects degraded (11.005%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:49.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[88498]: pgmap v234: 161 pgs: 19 active+undersized+degraded, 17 peering, 33 active+undersized, 92 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-05-20T23:03:49.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:49.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:49.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:49.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:49 smithi055 ceph-mon[88498]: Health check failed: Degraded data redundancy: 69/627 objects degraded (11.005%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:50.641 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:50.641 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:50.641 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:50 smithi138 ceph-mon[67823]: pgmap v235: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-05-20T23:03:50.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:50 smithi055 ceph-mon[86116]: pgmap v235: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-05-20T23:03:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:03:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:50 smithi055 ceph-mon[88498]: pgmap v235: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-05-20T23:03:51.523 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:03:51.226+0000 7fedd61147c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-05-20T23:03:51.524 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:03:51.254+0000 7fedd61147c0 -1 osd.7 149 log_to_monitors true 2024-05-20T23:03:51.797 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:51 smithi138 ceph-mon[67823]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-05-20T23:03:51.797 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:51 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:51.797 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:51 smithi138 ceph-mon[67823]: from='osd.7 [v2:172.21.15.138:6824/2552676636,v1:172.21.15.138:6825/2552676636]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T23:03:51.797 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:51 smithi138 ceph-mon[67823]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T23:03:51.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:51 smithi055 ceph-mon[86116]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-05-20T23:03:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:51 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:51 smithi055 ceph-mon[86116]: from='osd.7 [v2:172.21.15.138:6824/2552676636,v1:172.21.15.138:6825/2552676636]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T23:03:51.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:51 smithi055 ceph-mon[86116]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T23:03:51.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:51 smithi055 ceph-mon[88498]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-05-20T23:03:51.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:51 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:03:51.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:51 smithi055 ceph-mon[88498]: from='osd.7 [v2:172.21.15.138:6824/2552676636,v1:172.21.15.138:6825/2552676636]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T23:03:51.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:51 smithi055 ceph-mon[88498]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T23:03:52.099 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:03:51 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:03:51.805+0000 7fedc756b700 -1 osd.7 149 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T23:03:53.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:52 smithi138 ceph-mon[67823]: pgmap v236: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-05-20T23:03:53.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:52 smithi138 ceph-mon[67823]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T23:03:53.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:52 smithi138 ceph-mon[67823]: from='osd.7 [v2:172.21.15.138:6824/2552676636,v1:172.21.15.138:6825/2552676636]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:53.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:52 smithi138 ceph-mon[67823]: osdmap e152: 8 total, 7 up, 8 in 2024-05-20T23:03:53.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:52 smithi138 ceph-mon[67823]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:53.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:53.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:52 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:53.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[86116]: pgmap v236: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-05-20T23:03:53.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[86116]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T23:03:53.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[86116]: from='osd.7 [v2:172.21.15.138:6824/2552676636,v1:172.21.15.138:6825/2552676636]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:53.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[86116]: osdmap e152: 8 total, 7 up, 8 in 2024-05-20T23:03:53.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[86116]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:53.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:53.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:53.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[88498]: pgmap v236: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-05-20T23:03:53.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[88498]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T23:03:53.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[88498]: from='osd.7 [v2:172.21.15.138:6824/2552676636,v1:172.21.15.138:6825/2552676636]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:53.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[88498]: osdmap e152: 8 total, 7 up, 8 in 2024-05-20T23:03:53.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[88498]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-05-20T23:03:53.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:53.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:52 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:54.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:53 smithi138 ceph-mon[67823]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:03:54.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:53 smithi138 ceph-mon[67823]: osd.7 [v2:172.21.15.138:6824/2552676636,v1:172.21.15.138:6825/2552676636] boot 2024-05-20T23:03:54.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:53 smithi138 ceph-mon[67823]: osdmap e153: 8 total, 8 up, 8 in 2024-05-20T23:03:54.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:53 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T23:03:54.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:53 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:54.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:53 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:54.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[86116]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:03:54.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[86116]: osd.7 [v2:172.21.15.138:6824/2552676636,v1:172.21.15.138:6825/2552676636] boot 2024-05-20T23:03:54.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[86116]: osdmap e153: 8 total, 8 up, 8 in 2024-05-20T23:03:54.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T23:03:54.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:54.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:54.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[88498]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T23:03:54.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[88498]: osd.7 [v2:172.21.15.138:6824/2552676636,v1:172.21.15.138:6825/2552676636] boot 2024-05-20T23:03:54.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[88498]: osdmap e153: 8 total, 8 up, 8 in 2024-05-20T23:03:54.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T23:03:54.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:54.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:53 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:54 smithi138 ceph-mon[67823]: pgmap v239: 161 pgs: 10 peering, 23 active+undersized+degraded, 36 active+undersized, 92 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-05-20T23:03:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:54 smithi138 ceph-mon[67823]: osdmap e154: 8 total, 8 up, 8 in 2024-05-20T23:03:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:54 smithi138 ceph-mon[67823]: Health check update: Degraded data redundancy: 74/627 objects degraded (11.802%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:55.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:54 smithi055 ceph-mon[86116]: pgmap v239: 161 pgs: 10 peering, 23 active+undersized+degraded, 36 active+undersized, 92 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-05-20T23:03:55.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:54 smithi055 ceph-mon[86116]: osdmap e154: 8 total, 8 up, 8 in 2024-05-20T23:03:55.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:54 smithi055 ceph-mon[86116]: Health check update: Degraded data redundancy: 74/627 objects degraded (11.802%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:55.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:54 smithi055 ceph-mon[88498]: pgmap v239: 161 pgs: 10 peering, 23 active+undersized+degraded, 36 active+undersized, 92 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 74/627 objects degraded (11.802%) 2024-05-20T23:03:55.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:54 smithi055 ceph-mon[88498]: osdmap e154: 8 total, 8 up, 8 in 2024-05-20T23:03:55.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:54 smithi055 ceph-mon[88498]: Health check update: Degraded data redundancy: 74/627 objects degraded (11.802%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T23:03:56.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:56 smithi138 ceph-mon[67823]: pgmap v241: 161 pgs: 2 activating, 10 peering, 21 active+undersized+degraded, 35 active+undersized, 93 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 72/627 objects degraded (11.483%) 2024-05-20T23:03:57.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:56 smithi055 ceph-mon[86116]: pgmap v241: 161 pgs: 2 activating, 10 peering, 21 active+undersized+degraded, 35 active+undersized, 93 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 72/627 objects degraded (11.483%) 2024-05-20T23:03:57.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:56 smithi055 ceph-mon[88498]: pgmap v241: 161 pgs: 2 activating, 10 peering, 21 active+undersized+degraded, 35 active+undersized, 93 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 72/627 objects degraded (11.483%) 2024-05-20T23:03:57.732 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:03:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:03:57] "GET /metrics HTTP/1.1" 200 38290 "" "Prometheus/2.43.0" 2024-05-20T23:03:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all osd 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-20T23:03:58.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-20T23:03:58.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-20T23:03:58.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-20T23:03:58.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-20T23:03:58.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-20T23:03:58.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-20T23:03:58.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:58.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all osd 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-20T23:03:58.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-20T23:03:58.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-20T23:03:58.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-20T23:03:58.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-20T23:03:58.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-20T23:03:58.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-20T23:03:58.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-20T23:03:58.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-20T23:03:58.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-20T23:03:58.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-20T23:03:58.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-20T23:03:58.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-20T23:03:58.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: Upgrade: Setting require_osd_release to 19 squid 2024-05-20T23:03:58.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-20T23:03:58.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-20T23:03:58.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-20T23:03:58.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-20T23:03:58.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: Upgrade: Setting require_osd_release to 19 squid 2024-05-20T23:03:58.239 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:57 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all osd 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-20T23:03:58.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-20T23:03:58.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-20T23:03:58.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-20T23:03:58.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-20T23:03:58.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: Upgrade: Setting require_osd_release to 19 squid 2024-05-20T23:03:58.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:57 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-20T23:03:59.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: pgmap v242: 161 pgs: 2 activating, 10 peering, 149 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 72/627 objects degraded (11.483%), 21 pgs degraded) 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: osdmap e155: 8 total, 8 up, 8 in 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mds 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all nfs 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: pgmap v242: 161 pgs: 2 activating, 10 peering, 149 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 72/627 objects degraded (11.483%), 21 pgs degraded) 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: osdmap e155: 8 total, 8 up, 8 in 2024-05-20T23:03:59.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mds 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all nfs 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:03:59.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:03:59.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:03:59.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:03:59.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:03:59.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-20T23:03:59.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:03:59.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T23:03:59.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:03:59.239 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.240 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:59.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:59.247 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:59.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:59.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:59.248 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:58 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: pgmap v242: 161 pgs: 2 activating, 10 peering, 149 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:03:59.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 72/627 objects degraded (11.483%), 21 pgs degraded) 2024-05-20T23:03:59.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-20T23:03:59.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-20T23:03:59.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: osdmap e155: 8 total, 8 up, 8 in 2024-05-20T23:03:59.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mds 2024-05-20T23:03:59.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nfs 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:03:59.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T23:03:59.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:03:59.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:59.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:59.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:59.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:03:59.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:03:59.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:03:59.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:03:59.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:58 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:00.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:59 smithi138 ceph-mon[67823]: Upgrade: Finalizing container_image settings 2024-05-20T23:04:00.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:59 smithi138 ceph-mon[67823]: Upgrade: Complete! 2024-05-20T23:04:00.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:03:59 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:00.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:59 smithi055 ceph-mon[86116]: Upgrade: Finalizing container_image settings 2024-05-20T23:04:00.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:59 smithi055 ceph-mon[86116]: Upgrade: Complete! 2024-05-20T23:04:00.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:03:59 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:00.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:59 smithi055 ceph-mon[88498]: Upgrade: Finalizing container_image settings 2024-05-20T23:04:00.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:59 smithi055 ceph-mon[88498]: Upgrade: Complete! 2024-05-20T23:04:00.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:03:59 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:01.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:01 smithi055 ceph-mon[86116]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-05-20T23:04:01.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:01 smithi055 ceph-mon[88498]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-05-20T23:04:01.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:01 smithi138 ceph-mon[67823]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-05-20T23:04:02.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:02 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:02.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:02 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:02.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:02 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:03.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:03 smithi138 ceph-mon[67823]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:03.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:03 smithi055 ceph-mon[86116]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:03.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:03 smithi055 ceph-mon[88498]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:05.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:05 smithi138 ceph-mon[67823]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T23:04:05.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:05 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:05.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:05 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:05.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:05 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:05.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:05 smithi055 ceph-mon[88498]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T23:04:05.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:05 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:05.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:05 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:05.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:05 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:05.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:05 smithi055 ceph-mon[86116]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-20T23:04:05.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:05 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:05.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:05 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:05.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:05 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:07.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:07 smithi138 ceph-mon[67823]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:04:07.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:07 smithi055 ceph-mon[86116]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:04:07.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:07 smithi055 ceph-mon[88498]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:04:07.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:04:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:04:07] "GET /metrics HTTP/1.1" 200 38301 "" "Prometheus/2.43.0" 2024-05-20T23:04:09.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:09 smithi138 ceph-mon[67823]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:04:09.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:09 smithi055 ceph-mon[86116]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:04:09.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:09 smithi055 ceph-mon[88498]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:04:11.215 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T23:04:11.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:11 smithi138 ceph-mon[67823]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-05-20T23:04:11.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:11 smithi055 ceph-mon[88498]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-05-20T23:04:11.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:11 smithi055 ceph-mon[86116]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-05-20T23:04:12.190 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:12 smithi055 ceph-mon[86116]: from='client.64540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:12.190 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:12 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:12.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:12 smithi055 ceph-mon[88498]: from='client.64540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:12.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:12 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:12.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:12 smithi138 ceph-mon[67823]: from='client.64540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:12.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:12 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:13.187 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:13 smithi055 ceph-mon[86116]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:13.187 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:13 smithi055 ceph-mon[88498]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:13.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:13 smithi138 ceph-mon[67823]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:13.441 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:04:13.441 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (9m) 2m ago 31m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:04:13.441 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (9m) 20s ago 31m 81.3M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:04:13.441 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (8m) 2m ago 30m 64.0M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:04:13.441 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (10m) 20s ago 36m 455M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:04:13.441 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (20m) 2m ago 37m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:04:13.441 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (7m) 2m ago 38m 50.9M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:04:13.441 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (7m) 20s ago 36m 49.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:04:13.441 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (6m) 2m ago 36m 41.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (10m) 2m ago 32m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (10m) 20s ago 32m 15.2M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (4m) 2m ago 35m 52.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (4m) 2m ago 35m 52.5M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (3m) 2m ago 35m 49.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (2m) 2m ago 34m 12.7M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (2m) 20s ago 34m 48.1M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 c32ea1e7b587 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (96s) 20s ago 33m 52.6M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 b35ee64d41ac 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (60s) 20s ago 33m 51.4M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 ceee3eb06ecb 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (25s) 20s ago 33m 11.7M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 e41094f18f45 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (9m) 20s ago 32m 79.6M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (30m) 2m ago 30m 77.6M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:04:13.442 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (30m) 20s ago 30m 82.0M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:04:14.017 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-05-20T23:04:15.232 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:15 smithi055 ceph-mon[88498]: from='client.54615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:15.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:15 smithi055 ceph-mon[88498]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:15.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:15 smithi055 ceph-mon[86116]: from='client.54615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:15.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:15 smithi055 ceph-mon[86116]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:15.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:15 smithi138 ceph-mon[67823]: from='client.54615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:15.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:15 smithi138 ceph-mon[67823]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:16.275 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:04:16.946 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-05-20T23:04:17.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:17 smithi055 ceph-mon[86116]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:17.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:17 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/595015043' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:17.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:17 smithi055 ceph-mon[88498]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:17.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:17 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/595015043' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:17.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:17 smithi138 ceph-mon[67823]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:17.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:17 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/595015043' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:17.570 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:04:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:04:17] "GET /metrics HTTP/1.1" 200 38301 "" "Prometheus/2.43.0" 2024-05-20T23:04:19.225 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)" 2024-05-20T23:04:19.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:19 smithi138 ceph-mon[67823]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:19.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:19 smithi055 ceph-mon[86116]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:19.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:19 smithi055 ceph-mon[88498]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:19.848 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade status' 2024-05-20T23:04:20.163 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:20 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/705630558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:20.163 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:20.163 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:20 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/705630558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:20.163 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:20.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:20 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/705630558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:20.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:21.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:21 smithi138 ceph-mon[67823]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:21.447 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:21 smithi055 ceph-mon[86116]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:21.447 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:21 smithi055 ceph-mon[88498]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:21.903 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:04:21.903 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": null, 2024-05-20T23:04:21.903 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": false, 2024-05-20T23:04:21.904 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "", 2024-05-20T23:04:21.904 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:04:21.904 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": null, 2024-05-20T23:04:21.904 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "", 2024-05-20T23:04:21.904 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:04:21.904 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:04:22.214 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:22 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:22.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:22 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:22.533 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T23:04:22.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:22 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:23.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:23 smithi055 ceph-mon[86116]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:23.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:23 smithi055 ceph-mon[86116]: from='client.64564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:23 smithi055 ceph-mon[88498]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:23 smithi055 ceph-mon[88498]: from='client.64564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:23.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:23 smithi138 ceph-mon[67823]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:23.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:23 smithi138 ceph-mon[67823]: from='client.64564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:24.553 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T23:04:25.125 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:25 smithi055 ceph-mon[86116]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:25.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:25 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2240037091' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:04:25.189 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-05-20T23:04:25.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:25 smithi055 ceph-mon[88498]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:25.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:25 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2240037091' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:04:25.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:25 smithi138 ceph-mon[67823]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:25.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:25 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2240037091' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:04:27.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:27 smithi055 ceph-mon[86116]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:27.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:27 smithi055 ceph-mon[88498]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:27.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:04:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:04:27] "GET /metrics HTTP/1.1" 200 38300 "" "Prometheus/2.43.0" 2024-05-20T23:04:27.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:27 smithi138 ceph-mon[67823]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:29.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:29 smithi055 ceph-mon[88498]: from='client.54645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:29.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:29 smithi055 ceph-mon[88498]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:29.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:29 smithi055 ceph-mon[86116]: from='client.54645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:29.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:29 smithi055 ceph-mon[86116]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:29.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:29 smithi138 ceph-mon[67823]: from='client.54645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:29.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:29 smithi138 ceph-mon[67823]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:29.737 INFO:teuthology.orchestra.run.smithi055.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:04:30.383 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-20T23:04:30.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[88498]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:30.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[88498]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:04:30.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:30.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:30.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:30.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:30.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[86116]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[86116]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:04:30.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:30.984 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:30 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:31.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:30 smithi138 ceph-mon[67823]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:31.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:30 smithi138 ceph-mon[67823]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:04:31.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:30 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:31.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:30 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:31.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:30 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:31.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:30 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:31.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:30 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:31.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:31 smithi055 ceph-mon[86116]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:04:31.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:31 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:31.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:31 smithi055 ceph-mon[88498]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:04:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:31 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:32.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:31 smithi138 ceph-mon[67823]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:04:32.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:31 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:32.493 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:04:32.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:32 smithi055 ceph-mon[88498]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:32.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:32 smithi055 ceph-mon[86116]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:33.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:32 smithi138 ceph-mon[67823]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:33.186 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:04:33.186 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (9m) 2m ago 31m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (9m) 40s ago 31m 81.3M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (9m) 2m ago 31m 64.0M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (11m) 40s ago 36m 455M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (21m) 2m ago 38m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (7m) 2m ago 38m 50.9M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (8m) 40s ago 36m 49.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (6m) 2m ago 37m 41.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (10m) 2m ago 32m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (10m) 40s ago 32m 15.2M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (5m) 2m ago 36m 52.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (4m) 2m ago 35m 52.5M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (3m) 2m ago 35m 49.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (2m) 2m ago 35m 12.7M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:04:33.187 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (2m) 40s ago 34m 48.1M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 c32ea1e7b587 2024-05-20T23:04:33.188 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (115s) 40s ago 34m 52.6M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 b35ee64d41ac 2024-05-20T23:04:33.188 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (80s) 40s ago 33m 51.4M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 ceee3eb06ecb 2024-05-20T23:04:33.188 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (45s) 40s ago 33m 11.7M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 e41094f18f45 2024-05-20T23:04:33.188 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (10m) 40s ago 32m 79.6M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:04:33.188 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (31m) 2m ago 31m 77.6M - 17.2.0 e1d6a67b021e b6e07ba8fd33 2024-05-20T23:04:33.188 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (31m) 40s ago 31m 82.0M - 17.2.0 e1d6a67b021e 6e55bfba53d3 2024-05-20T23:04:33.620 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:04:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-20T23:04:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 13 2024-05-20T23:04:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:04:33.622 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:04:33.979 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:04:33.979 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T23:04:33.980 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T23:04:33.980 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-05-20T23:04:33.980 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:04:33.980 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "0/2 daemons upgraded", 2024-05-20T23:04:33.980 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading rgw daemons", 2024-05-20T23:04:33.980 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:04:33.980 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:04:34.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='client.64582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:34.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='client.64588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:34.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='client.64594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/3323931114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:33 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='client.64582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:34.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='client.64588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:34.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='client.64594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:34.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:34.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/3323931114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.235 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='client.64582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='client.64588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='client.64594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/3323931114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:34.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:33 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:04:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:04:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mgr 2024-05-20T23:04:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mon 2024-05-20T23:04:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all crash 2024-05-20T23:04:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all osd 2024-05-20T23:04:35.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mds 2024-05-20T23:04:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: from='client.64606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:35.107 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:35.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:35.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:04:35.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:04:35.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mgr 2024-05-20T23:04:35.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mon 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all crash 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all osd 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mds 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: from='client.64606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mgr 2024-05-20T23:04:35.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mon 2024-05-20T23:04:35.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all crash 2024-05-20T23:04:35.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all osd 2024-05-20T23:04:35.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mds 2024-05-20T23:04:35.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: from='client.64606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:04:35.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:36.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[88498]: Upgrade: Updating rgw.foo.smithi055.ymnqng (1/2) 2024-05-20T23:04:36.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:36.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T23:04:36.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:36.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[88498]: Deploying daemon rgw.foo.smithi055.ymnqng on smithi055 2024-05-20T23:04:36.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[86116]: Upgrade: Updating rgw.foo.smithi055.ymnqng (1/2) 2024-05-20T23:04:36.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:36.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T23:04:36.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:36.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:36 smithi055 ceph-mon[86116]: Deploying daemon rgw.foo.smithi055.ymnqng on smithi055 2024-05-20T23:04:36.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:36 smithi138 ceph-mon[67823]: Upgrade: Updating rgw.foo.smithi055.ymnqng (1/2) 2024-05-20T23:04:36.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:36 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:36.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:36 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi055.ymnqng", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T23:04:36.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:36 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:36.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:36 smithi138 ceph-mon[67823]: Deploying daemon rgw.foo.smithi055.ymnqng on smithi055 2024-05-20T23:04:37.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:37 smithi055 ceph-mon[86116]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:37.344 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:04:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:04:37] "GET /metrics HTTP/1.1" 200 38301 "" "Prometheus/2.43.0" 2024-05-20T23:04:37.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:37 smithi055 ceph-mon[88498]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:37.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:37 smithi138 ceph-mon[67823]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:39.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:39 smithi055 ceph-mon[86116]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:39.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:39 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:39.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:39 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:39 smithi055 ceph-mon[88498]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:39 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:39.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:39 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:39.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:39 smithi138 ceph-mon[67823]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:04:39.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:39 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:39.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:39 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:41.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:41 smithi055 ceph-mon[88498]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:41.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:41.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T23:04:41.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:41 smithi055 ceph-mon[86116]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T23:04:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:41.513 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:41 smithi138 ceph-mon[67823]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:04:41.513 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:41.513 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi138.sfpuja", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T23:04:41.513 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:42.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:42 smithi138 ceph-mon[67823]: Upgrade: Updating rgw.foo.smithi138.sfpuja (2/2) 2024-05-20T23:04:42.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:42 smithi138 ceph-mon[67823]: Deploying daemon rgw.foo.smithi138.sfpuja on smithi138 2024-05-20T23:04:42.303 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:42 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:42.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:42 smithi055 ceph-mon[88498]: Upgrade: Updating rgw.foo.smithi138.sfpuja (2/2) 2024-05-20T23:04:42.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:42 smithi055 ceph-mon[88498]: Deploying daemon rgw.foo.smithi138.sfpuja on smithi138 2024-05-20T23:04:42.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:42 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:42.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:42 smithi055 ceph-mon[86116]: Upgrade: Updating rgw.foo.smithi138.sfpuja (2/2) 2024-05-20T23:04:42.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:42 smithi055 ceph-mon[86116]: Deploying daemon rgw.foo.smithi138.sfpuja on smithi138 2024-05-20T23:04:42.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:42 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:43.425 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:43 smithi138 ceph-mon[67823]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 27 op/s 2024-05-20T23:04:43.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:43 smithi055 ceph-mon[86116]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 27 op/s 2024-05-20T23:04:43.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:43 smithi055 ceph-mon[88498]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 27 op/s 2024-05-20T23:04:45.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:45 smithi138 ceph-mon[67823]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-20T23:04:45.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:45.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:45.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:45 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:45.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:45 smithi055 ceph-mon[86116]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-20T23:04:45.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:45.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:45.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:45 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:45.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:45 smithi055 ceph-mon[88498]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 0 B/s wr, 71 op/s 2024-05-20T23:04:45.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:45.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:45.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:45 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:46.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:46 smithi138 ceph-mon[67823]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-05-20T23:04:46.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:46 smithi055 ceph-mon[86116]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-05-20T23:04:46.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:46 smithi055 ceph-mon[88498]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-05-20T23:04:47.576 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:04:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:04:47] "GET /metrics HTTP/1.1" 200 38301 "" "Prometheus/2.43.0" 2024-05-20T23:04:48.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:48 smithi055 ceph-mon[88498]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 85 B/s wr, 143 op/s 2024-05-20T23:04:48.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:48 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:48.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:48 smithi055 ceph-mon[86116]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 85 B/s wr, 143 op/s 2024-05-20T23:04:48.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:48 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:49.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:48 smithi138 ceph-mon[67823]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 85 B/s wr, 143 op/s 2024-05-20T23:04:49.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:48 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:49.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:49.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:49.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:49.985 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:49 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:49.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:49.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:49.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:49.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:49 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:50.006 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.006 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.007 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.007 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:49 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:04:50.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[88498]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-05-20T23:04:50.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[86116]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-05-20T23:04:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:50.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:51.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:50 smithi138 ceph-mon[67823]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-05-20T23:04:51.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:51.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:51.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:51.106 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:51.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:51 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:51.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:51 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:52.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:51 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:04:52.983 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:52 smithi138 ceph-mon[67823]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-05-20T23:04:53.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:52 smithi055 ceph-mon[86116]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-05-20T23:04:53.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:52 smithi055 ceph-mon[88498]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 168 op/s 2024-05-20T23:04:54.047 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.047 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.047 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.047 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.047 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:54.047 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:54.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:54.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi055.ymnqng"}]: dispatch 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi055.ymnqng"}]': finished 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi138.sfpuja"}]: dispatch 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi138.sfpuja"}]': finished 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.357 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:04:54.358 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:04:54.359 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.360 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:04:54.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:04:54.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:54.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:54.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:54.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:54.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:54.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:54.361 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:54 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi055.ymnqng"}]: dispatch 2024-05-20T23:04:54.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi055.ymnqng"}]': finished 2024-05-20T23:04:54.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi138.sfpuja"}]: dispatch 2024-05-20T23:04:54.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi138.sfpuja"}]': finished 2024-05-20T23:04:54.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:54.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi055.ymnqng"}]: dispatch 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi055.ymnqng"}]': finished 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi138.sfpuja"}]: dispatch 2024-05-20T23:04:54.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi138.sfpuja"}]': finished 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.487 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:04:54.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:54.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T23:04:54.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T23:04:54.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:54.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:54.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:54.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:54.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:04:54.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:04:54.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:04:54.494 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:54 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:55.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all rgw 2024-05-20T23:04:55.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T23:04:55.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T23:04:55.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nfs 2024-05-20T23:04:55.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:04:55.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: Upgrade: Finalizing container_image settings 2024-05-20T23:04:55.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: Upgrade: Complete! 2024-05-20T23:04:55.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: Checking dashboard <-> RGW credentials 2024-05-20T23:04:55.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 170 B/s wr, 141 op/s 2024-05-20T23:04:55.356 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:55 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:55.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all rgw 2024-05-20T23:04:55.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T23:04:55.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T23:04:55.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all nfs 2024-05-20T23:04:55.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:04:55.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: Upgrade: Finalizing container_image settings 2024-05-20T23:04:55.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: Upgrade: Complete! 2024-05-20T23:04:55.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: Checking dashboard <-> RGW credentials 2024-05-20T23:04:55.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 170 B/s wr, 141 op/s 2024-05-20T23:04:55.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all rgw 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all nfs 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all nvmeof 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: Upgrade: Finalizing container_image settings 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: Upgrade: Complete! 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: Checking dashboard <-> RGW credentials 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 170 B/s wr, 141 op/s 2024-05-20T23:04:55.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:55 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:04:57.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:57 smithi138 ceph-mon[67823]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 170 B/s wr, 97 op/s 2024-05-20T23:04:57.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:57 smithi055 ceph-mon[86116]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 170 B/s wr, 97 op/s 2024-05-20T23:04:57.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:57 smithi055 ceph-mon[88498]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 170 B/s wr, 97 op/s 2024-05-20T23:04:57.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:04:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:04:57] "GET /metrics HTTP/1.1" 200 38328 "" "Prometheus/2.43.0" 2024-05-20T23:04:59.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:04:59 smithi138 ceph-mon[67823]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 85 B/s wr, 86 op/s 2024-05-20T23:04:59.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:04:59 smithi055 ceph-mon[88498]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 85 B/s wr, 86 op/s 2024-05-20T23:04:59.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:04:59 smithi055 ceph-mon[86116]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 85 B/s wr, 86 op/s 2024-05-20T23:05:01.085 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:01 smithi055 ceph-mon[86116]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 85 B/s wr, 27 op/s 2024-05-20T23:05:01.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:01 smithi138 ceph-mon[67823]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 85 B/s wr, 27 op/s 2024-05-20T23:05:01.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:01 smithi055 ceph-mon[88498]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 85 B/s wr, 27 op/s 2024-05-20T23:05:02.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:02 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:02.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:02 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:02.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:02 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:03.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:03 smithi138 ceph-mon[67823]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-05-20T23:05:03.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:03 smithi055 ceph-mon[86116]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-05-20T23:05:03.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:03 smithi055 ceph-mon[88498]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-05-20T23:05:05.060 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T23:05:05.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:05 smithi138 ceph-mon[67823]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-20T23:05:05.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:05 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:05.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:05 smithi055 ceph-mon[86116]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-20T23:05:05.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:05 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:05.381 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:05 smithi055 ceph-mon[88498]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-20T23:05:05.381 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:05 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:06.375 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:06 smithi055 ceph-mon[86116]: from='client.64690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:06.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:06 smithi055 ceph-mon[88498]: from='client.64690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:06.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:06 smithi138 ceph-mon[67823]: from='client.64690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:07.133 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:05:07.133 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (10m) 18s ago 32m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (9m) 18s ago 32m 81.3M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (9m) 18s ago 31m 66.3M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (11m) 18s ago 37m 455M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (21m) 18s ago 38m 515M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (7m) 18s ago 38m 59.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (8m) 18s ago 37m 52.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (7m) 18s ago 37m 48.8M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (11m) 18s ago 33m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (11m) 18s ago 33m 15.0M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (5m) 18s ago 36m 60.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (5m) 18s ago 36m 61.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:05:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (4m) 18s ago 36m 56.1M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:05:07.135 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (3m) 18s ago 35m 52.0M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:05:07.135 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (3m) 18s ago 35m 52.7M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 c32ea1e7b587 2024-05-20T23:05:07.135 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (2m) 18s ago 34m 55.0M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 b35ee64d41ac 2024-05-20T23:05:07.135 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (114s) 18s ago 34m 54.6M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 ceee3eb06ecb 2024-05-20T23:05:07.135 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (79s) 18s ago 34m 46.2M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 e41094f18f45 2024-05-20T23:05:07.135 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (10m) 18s ago 33m 80.1M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:05:07.135 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (28s) 18s ago 31m 79.1M - 19.0.0-2695-g8a373da3 86045c3f4e75 b294685c019d 2024-05-20T23:05:07.135 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (22s) 18s ago 31m 77.4M - 19.0.0-2695-g8a373da3 86045c3f4e75 510f6b207ba0 2024-05-20T23:05:07.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:07 smithi055 ceph-mon[86116]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-20T23:05:07.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:05:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:05:07] "GET /metrics HTTP/1.1" 200 38331 "" "Prometheus/2.43.0" 2024-05-20T23:05:07.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:07 smithi055 ceph-mon[88498]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-20T23:05:07.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:07 smithi138 ceph-mon[67823]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-05-20T23:05:07.809 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-05-20T23:05:08.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:08 smithi055 ceph-mon[88498]: from='client.64696 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:08.387 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:08 smithi055 ceph-mon[86116]: from='client.64696 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:08.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:08 smithi138 ceph-mon[67823]: from='client.64696 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:09.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:09 smithi055 ceph-mon[86116]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2 op/s 2024-05-20T23:05:09.356 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:09 smithi055 ceph-mon[88498]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2 op/s 2024-05-20T23:05:09.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:09 smithi138 ceph-mon[67823]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2 op/s 2024-05-20T23:05:09.898 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:05:10.423 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:10 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/1186646407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:10.423 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:10 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/1186646407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:10.448 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-05-20T23:05:10.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:10 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/1186646407' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:11.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:11 smithi055 ceph-mon[86116]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:05:11.474 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:11 smithi055 ceph-mon[88498]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:05:11.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:11 smithi138 ceph-mon[67823]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-20T23:05:12.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:12 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:12.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:12 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:12.595 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)" 2024-05-20T23:05:12.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:12 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:12.605 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:05:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=cleanup t=2024-05-20T23:05:12.430944489Z level=info msg="Completed cleanup jobs" duration=2.525419ms 2024-05-20T23:05:13.257 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade status' 2024-05-20T23:05:13.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:13 smithi055 ceph-mon[88498]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:13.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:13 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/917981220' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:13.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:13 smithi055 ceph-mon[86116]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:13.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:13 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/917981220' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:13.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:13 smithi138 ceph-mon[67823]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:13.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:13 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/917981220' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:15.241 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:05:15.241 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": null, 2024-05-20T23:05:15.241 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": false, 2024-05-20T23:05:15.241 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "", 2024-05-20T23:05:15.241 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [], 2024-05-20T23:05:15.241 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": null, 2024-05-20T23:05:15.241 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "", 2024-05-20T23:05:15.241 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:05:15.241 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:05:15.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:15 smithi055 ceph-mon[86116]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:15.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:15 smithi055 ceph-mon[88498]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:15.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:15 smithi138 ceph-mon[67823]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:15.965 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T23:05:16.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:16 smithi055 ceph-mon[88498]: from='client.54786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:16.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:16 smithi055 ceph-mon[86116]: from='client.54786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:16.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:16 smithi138 ceph-mon[67823]: from='client.54786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:17.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:17 smithi055 ceph-mon[86116]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:17.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:17 smithi055 ceph-mon[88498]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:17.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:05:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:05:17] "GET /metrics HTTP/1.1" 200 38331 "" "Prometheus/2.43.0" 2024-05-20T23:05:17.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:17 smithi138 ceph-mon[67823]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:18.213 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T23:05:18.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:18 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/3753051533' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:05:18.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:18 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/3753051533' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:05:18.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:18 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/3753051533' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:05:18.779 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-20T23:05:19.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:19 smithi055 ceph-mon[86116]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:19.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:19 smithi055 ceph-mon[88498]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:19.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:19 smithi138 ceph-mon[67823]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:20.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:20.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:20.505 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:20.752 INFO:teuthology.orchestra.run.smithi055.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:05:21.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[86116]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[88498]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:05:21.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:05:21.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:21 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:21.527 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T23:05:21.538 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi055.front.sepia.ceph.com 2024-05-20T23:05:21.538 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-05-20T23:05:21.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:21 smithi138 ceph-mon[67823]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:21.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:21 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:21.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:21 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:05:21.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:21 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:05:21.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:21 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:05:21.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:21 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:22.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:22 smithi055 ceph-mon[86116]: from='client.64726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:22.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:22 smithi055 ceph-mon[86116]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:05:22.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:22 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:22.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:22 smithi055 ceph-mon[86116]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:05:22.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:22 smithi055 ceph-mon[88498]: from='client.64726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:22.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:22 smithi055 ceph-mon[88498]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:05:22.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:22 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:22.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:22 smithi055 ceph-mon[88498]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:05:22.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:22 smithi138 ceph-mon[67823]: from='client.64726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:22.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:22 smithi138 ceph-mon[67823]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:05:22.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:22 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:22.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:22 smithi138 ceph-mon[67823]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T23:05:23.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:23 smithi055 ceph-mon[86116]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:23.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:23 smithi055 ceph-mon[88498]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:23.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:23 smithi138 ceph-mon[67823]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:23.642 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:05:24.339 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:05:24.339 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (10m) 35s ago 32m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:05:24.339 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (10m) 35s ago 32m 81.3M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:05:24.339 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (9m) 35s ago 31m 66.3M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:05:24.339 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (11m) 35s ago 37m 455M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:05:24.339 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (22m) 35s ago 39m 515M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:05:24.339 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (8m) 35s ago 39m 59.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:05:24.339 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (9m) 35s ago 37m 52.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (7m) 35s ago 37m 48.8M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (11m) 35s ago 33m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (11m) 35s ago 33m 15.0M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (6m) 35s ago 37m 60.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (5m) 35s ago 36m 61.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (4m) 35s ago 36m 56.1M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (3m) 35s ago 35m 52.0M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (3m) 35s ago 35m 52.7M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 c32ea1e7b587 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (2m) 35s ago 35m 55.0M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 b35ee64d41ac 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (2m) 35s ago 34m 54.6M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 ceee3eb06ecb 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (96s) 35s ago 34m 46.2M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 e41094f18f45 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (11m) 35s ago 33m 80.1M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:05:24.340 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (45s) 35s ago 32m 79.1M - 19.0.0-2695-g8a373da3 86045c3f4e75 b294685c019d 2024-05-20T23:05:24.341 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (40s) 35s ago 32m 77.4M - 19.0.0-2695-g8a373da3 86045c3f4e75 510f6b207ba0 2024-05-20T23:05:24.771 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:05:24.771 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:05:24.771 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:05:24.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:05:24.771 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:05:24.771 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:05:24.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 15 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:05:24.772 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:05:25.178 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:05:25.178 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T23:05:25.178 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T23:05:25.178 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [ 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw", 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr", 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout: "osd", 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout: "mon" 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "15/21 daemons upgraded", 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Currently upgrading iscsi daemons", 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": false 2024-05-20T23:05:25.179 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='client.54804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='client.54810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.484 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/1849781331' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:25.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='client.54804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:25.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='client.54810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:25.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:05:25.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.486 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.487 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:25 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/1849781331' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:25.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='client.54804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:25.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='client.54810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:25.607 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:25 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/1849781331' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:05:25.611 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_OK 2024-05-20T23:05:26.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: from='client.54816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:26.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:05:26.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:05:26.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mgr 2024-05-20T23:05:26.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mon 2024-05-20T23:05:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all crash 2024-05-20T23:05:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all osd 2024-05-20T23:05:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all mds 2024-05-20T23:05:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all rgw 2024-05-20T23:05:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T23:05:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T23:05:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T23:05:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: from='client.54828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:26.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:26 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/3273555472' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:05:26.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: from='client.54816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mgr 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mon 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all crash 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all osd 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all mds 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all rgw 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T23:05:26.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T23:05:26.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: from='client.54828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:26.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/3273555472' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:05:26.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: from='client.54816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:26.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (squid) 2024-05-20T23:05:26.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T23:05:26.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mgr 2024-05-20T23:05:26.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mon 2024-05-20T23:05:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all crash 2024-05-20T23:05:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all osd 2024-05-20T23:05:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all mds 2024-05-20T23:05:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all rgw 2024-05-20T23:05:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T23:05:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-20T23:05:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T23:05:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: from='client.54828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:26.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:26 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/3273555472' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:05:27.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:27 smithi138 ceph-mon[67823]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:27.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:27 smithi138 ceph-mon[67823]: Upgrade: Updating iscsi.foo.smithi055.akoeuh 2024-05-20T23:05:27.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:27 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:27.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:27 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T23:05:27.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:27 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:05:27.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:27 smithi138 ceph-mon[67823]: Deploying daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T23:05:27.661 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:05:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:05:27] "GET /metrics HTTP/1.1" 200 38330 "" "Prometheus/2.43.0" 2024-05-20T23:05:27.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[86116]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:27.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[86116]: Upgrade: Updating iscsi.foo.smithi055.akoeuh 2024-05-20T23:05:27.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:27.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T23:05:27.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:05:27.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[86116]: Deploying daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T23:05:27.662 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[88498]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:27.662 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[88498]: Upgrade: Updating iscsi.foo.smithi055.akoeuh 2024-05-20T23:05:27.663 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:27.663 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi055.akoeuh", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-20T23:05:27.663 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:05:27.663 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:27 smithi055 ceph-mon[88498]: Deploying daemon iscsi.foo.smithi055.akoeuh on smithi055 2024-05-20T23:05:28.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:28 smithi138 ceph-mon[67823]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:28.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:28 smithi055 ceph-mon[86116]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:28.642 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:28 smithi055 ceph-mon[88498]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:30.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:30 smithi055 ceph-mon[86116]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:30.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:30 smithi055 ceph-mon[88498]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:31.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:30 smithi138 ceph-mon[67823]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:31.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:31 smithi138 ceph-mon[67823]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:31.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:31 smithi055 ceph-mon[86116]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:31.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:31 smithi055 ceph-mon[88498]: from='client.15762 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:05:32.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:32 smithi055 ceph-mon[86116]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:32.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:32 smithi055 ceph-mon[88498]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:33.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:32 smithi138 ceph-mon[67823]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:34.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:34 smithi055 ceph-mon[88498]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:34.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:34 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:34.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:34 smithi055 ceph-mon[86116]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:34.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:34 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:35.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:34 smithi138 ceph-mon[67823]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:35.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:34 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:36.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:36 smithi055 ceph-mon[86116]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:36.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:36 smithi055 ceph-mon[88498]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:37.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:36 smithi138 ceph-mon[67823]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:37.732 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:05:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:05:37] "GET /metrics HTTP/1.1" 200 38327 "" "Prometheus/2.43.0" 2024-05-20T23:05:38.950 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:38 smithi055 ceph-mon[86116]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:38.950 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:38 smithi055 ceph-mon[88498]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:39.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:38 smithi138 ceph-mon[67823]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:40.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:05:40 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T23:05:40.133+0000 7f063db4e700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed. 2024-05-20T23:05:41.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[86116]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[86116]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed. 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[86116]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[88498]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[88498]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed. 2024-05-20T23:05:41.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:41.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:05:41.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[88498]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-05-20T23:05:41.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:05:41.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:05:41.484 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:41 smithi138 ceph-mon[67823]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:05:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:41 smithi138 ceph-mon[67823]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed. 2024-05-20T23:05:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:05:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:41 smithi138 ceph-mon[67823]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-05-20T23:05:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:05:41.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:05:41.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:42.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:42 smithi055 ceph-mon[86116]: Health check failed: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-20T23:05:42.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:42 smithi055 ceph-mon[88498]: Health check failed: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-20T23:05:42.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:42 smithi138 ceph-mon[67823]: Health check failed: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-20T23:05:43.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:43 smithi055 ceph-mon[86116]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 679 B/s rd, 0 op/s 2024-05-20T23:05:43.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:43 smithi055 ceph-mon[88498]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 679 B/s rd, 0 op/s 2024-05-20T23:05:43.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:43 smithi138 ceph-mon[67823]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 679 B/s rd, 0 op/s 2024-05-20T23:05:45.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:45 smithi055 ceph-mon[86116]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 679 B/s rd, 0 op/s 2024-05-20T23:05:45.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:45 smithi055 ceph-mon[88498]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 679 B/s rd, 0 op/s 2024-05-20T23:05:45.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:45 smithi138 ceph-mon[67823]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 679 B/s rd, 0 op/s 2024-05-20T23:05:47.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:47 smithi055 ceph-mon[86116]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 679 B/s rd, 0 op/s 2024-05-20T23:05:47.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:47 smithi055 ceph-mon[88498]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 679 B/s rd, 0 op/s 2024-05-20T23:05:47.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:05:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:05:47] "GET /metrics HTTP/1.1" 200 38327 "" "Prometheus/2.43.0" 2024-05-20T23:05:47.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:47 smithi138 ceph-mon[67823]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 679 B/s rd, 0 op/s 2024-05-20T23:05:49.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:49 smithi055 ceph-mon[86116]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 194 B/s rd, 0 op/s 2024-05-20T23:05:49.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:49 smithi055 ceph-mon[88498]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 194 B/s rd, 0 op/s 2024-05-20T23:05:49.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:49 smithi138 ceph-mon[67823]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 194 B/s rd, 0 op/s 2024-05-20T23:05:50.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:50.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:50.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:50 smithi055 ceph-mon[86116]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 194 B/s rd, 0 op/s 2024-05-20T23:05:50.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:50.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:50.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:50 smithi055 ceph-mon[88498]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 194 B/s rd, 0 op/s 2024-05-20T23:05:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:05:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:05:50.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:50 smithi138 ceph-mon[67823]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 194 B/s rd, 0 op/s 2024-05-20T23:05:51.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:51 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/388704489' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T23:05:51.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:51 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/388704489' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T23:05:51.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:51 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/388704489' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T23:05:52.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:52 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/4111315205' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2923796288"}]: dispatch 2024-05-20T23:05:52.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:52 smithi138 ceph-mon[67823]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T23:05:52.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:52 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/4111315205' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2923796288"}]: dispatch 2024-05-20T23:05:52.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:52 smithi055 ceph-mon[86116]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T23:05:52.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:52 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/4111315205' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2923796288"}]: dispatch 2024-05-20T23:05:52.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:52 smithi055 ceph-mon[88498]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-05-20T23:05:53.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:53 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/4111315205' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2923796288"}]': finished 2024-05-20T23:05:53.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:53 smithi138 ceph-mon[67823]: osdmap e156: 8 total, 8 up, 8 in 2024-05-20T23:05:53.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:53 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/337870299' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3386919359"}]: dispatch 2024-05-20T23:05:53.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:53 smithi138 ceph-mon[67823]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3386919359"}]: dispatch 2024-05-20T23:05:53.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:53 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/4111315205' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2923796288"}]': finished 2024-05-20T23:05:53.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:53 smithi055 ceph-mon[86116]: osdmap e156: 8 total, 8 up, 8 in 2024-05-20T23:05:53.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:53 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/337870299' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3386919359"}]: dispatch 2024-05-20T23:05:53.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:53 smithi055 ceph-mon[86116]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3386919359"}]: dispatch 2024-05-20T23:05:53.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:53 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/4111315205' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2923796288"}]': finished 2024-05-20T23:05:53.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:53 smithi055 ceph-mon[88498]: osdmap e156: 8 total, 8 up, 8 in 2024-05-20T23:05:53.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:53 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/337870299' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3386919359"}]: dispatch 2024-05-20T23:05:53.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:53 smithi055 ceph-mon[88498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3386919359"}]: dispatch 2024-05-20T23:05:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:54 smithi138 ceph-mon[67823]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3386919359"}]': finished 2024-05-20T23:05:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:54 smithi138 ceph-mon[67823]: osdmap e157: 8 total, 8 up, 8 in 2024-05-20T23:05:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:54 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2872329602' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2841615960"}]: dispatch 2024-05-20T23:05:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:54 smithi138 ceph-mon[67823]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2841615960"}]: dispatch 2024-05-20T23:05:54.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:54 smithi138 ceph-mon[67823]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-05-20T23:05:54.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[86116]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3386919359"}]': finished 2024-05-20T23:05:54.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[86116]: osdmap e157: 8 total, 8 up, 8 in 2024-05-20T23:05:54.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2872329602' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2841615960"}]: dispatch 2024-05-20T23:05:54.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[86116]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2841615960"}]: dispatch 2024-05-20T23:05:54.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[86116]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-05-20T23:05:54.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[88498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6800/3386919359"}]': finished 2024-05-20T23:05:54.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[88498]: osdmap e157: 8 total, 8 up, 8 in 2024-05-20T23:05:54.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2872329602' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2841615960"}]: dispatch 2024-05-20T23:05:54.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[88498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2841615960"}]: dispatch 2024-05-20T23:05:54.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:54 smithi055 ceph-mon[88498]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-05-20T23:05:55.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:55 smithi055 ceph-mon[86116]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2841615960"}]': finished 2024-05-20T23:05:55.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:55 smithi055 ceph-mon[86116]: osdmap e158: 8 total, 8 up, 8 in 2024-05-20T23:05:55.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:55 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/1060122084' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/748655252"}]: dispatch 2024-05-20T23:05:55.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:55 smithi055 ceph-mon[88498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2841615960"}]': finished 2024-05-20T23:05:55.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:55 smithi055 ceph-mon[88498]: osdmap e158: 8 total, 8 up, 8 in 2024-05-20T23:05:55.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:55 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/1060122084' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/748655252"}]: dispatch 2024-05-20T23:05:55.998 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:05:56.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:55 smithi138 ceph-mon[67823]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/2841615960"}]': finished 2024-05-20T23:05:56.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:55 smithi138 ceph-mon[67823]: osdmap e158: 8 total, 8 up, 8 in 2024-05-20T23:05:56.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:55 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/1060122084' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/748655252"}]: dispatch 2024-05-20T23:05:56.348 INFO:teuthology.orchestra.run.smithi055.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed." 2024-05-20T23:05:56.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/1060122084' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/748655252"}]': finished 2024-05-20T23:05:56.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[88498]: osdmap e159: 8 total, 8 up, 8 in 2024-05-20T23:05:56.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2736060322' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3386919359"}]: dispatch 2024-05-20T23:05:56.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[88498]: from='client.54894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:56.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[88498]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:05:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/1060122084' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/748655252"}]': finished 2024-05-20T23:05:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[86116]: osdmap e159: 8 total, 8 up, 8 in 2024-05-20T23:05:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2736060322' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3386919359"}]: dispatch 2024-05-20T23:05:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[86116]: from='client.54894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:56.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:56 smithi055 ceph-mon[86116]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:05:57.092 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T23:05:57.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:56 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/1060122084' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/748655252"}]': finished 2024-05-20T23:05:57.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:56 smithi138 ceph-mon[67823]: osdmap e159: 8 total, 8 up, 8 in 2024-05-20T23:05:57.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:56 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2736060322' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3386919359"}]: dispatch 2024-05-20T23:05:57.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:56 smithi138 ceph-mon[67823]: from='client.54894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:57.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:56 smithi138 ceph-mon[67823]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-05-20T23:05:57.412 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:05:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:05:57] "GET /metrics HTTP/1.1" 200 38410 "" "Prometheus/2.43.0" 2024-05-20T23:05:57.677 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[88498]: from='client.54900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:57.677 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2736060322' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3386919359"}]': finished 2024-05-20T23:05:57.677 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[88498]: osdmap e160: 8 total, 8 up, 8 in 2024-05-20T23:05:57.677 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2048793620' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/107297477"}]: dispatch 2024-05-20T23:05:57.677 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[88498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/107297477"}]: dispatch 2024-05-20T23:05:57.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[86116]: from='client.54900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:57.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2736060322' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3386919359"}]': finished 2024-05-20T23:05:57.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[86116]: osdmap e160: 8 total, 8 up, 8 in 2024-05-20T23:05:57.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2048793620' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/107297477"}]: dispatch 2024-05-20T23:05:57.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:57 smithi055 ceph-mon[86116]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/107297477"}]: dispatch 2024-05-20T23:05:58.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:57 smithi138 ceph-mon[67823]: from='client.54900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:58.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:57 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2736060322' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:6801/3386919359"}]': finished 2024-05-20T23:05:58.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:57 smithi138 ceph-mon[67823]: osdmap e160: 8 total, 8 up, 8 in 2024-05-20T23:05:58.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:57 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2048793620' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/107297477"}]: dispatch 2024-05-20T23:05:58.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:57 smithi138 ceph-mon[67823]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/107297477"}]: dispatch 2024-05-20T23:05:58.907 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:58 smithi055 ceph-mon[86116]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/107297477"}]': finished 2024-05-20T23:05:58.907 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:58 smithi055 ceph-mon[86116]: osdmap e161: 8 total, 8 up, 8 in 2024-05-20T23:05:58.907 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:58 smithi055 ceph-mon[86116]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:58.907 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:58 smithi055 ceph-mon[88498]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/107297477"}]': finished 2024-05-20T23:05:58.907 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:58 smithi055 ceph-mon[88498]: osdmap e161: 8 total, 8 up, 8 in 2024-05-20T23:05:58.907 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:58 smithi055 ceph-mon[88498]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:58 smithi138 ceph-mon[67823]: from='client.? ' entity='client.iscsi.foo.smithi055.akoeuh' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.55:0/107297477"}]': finished 2024-05-20T23:05:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:58 smithi138 ceph-mon[67823]: osdmap e161: 8 total, 8 up, 8 in 2024-05-20T23:05:59.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:58 smithi138 ceph-mon[67823]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:05:59.122 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:05:59.122 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (11m) 70s ago 33m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:05:59.122 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (10m) 70s ago 33m 81.3M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:05:59.122 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (10m) 70s ago 32m 66.3M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:05:59.122 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (12m) 70s ago 38m 455M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:05:59.123 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (22m) 70s ago 39m 515M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:05:59.123 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (8m) 70s ago 39m 59.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:05:59.123 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (9m) 70s ago 38m 52.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:05:59.123 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (8m) 70s ago 38m 48.8M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:05:59.123 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (12m) 70s ago 34m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:05:59.124 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (12m) 70s ago 34m 15.0M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:05:59.124 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (6m) 70s ago 37m 60.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:05:59.124 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (6m) 70s ago 37m 61.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:05:59.124 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (5m) 70s ago 36m 56.1M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:05:59.124 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (4m) 70s ago 36m 52.0M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:05:59.124 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (3m) 70s ago 36m 52.7M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 c32ea1e7b587 2024-05-20T23:05:59.125 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (3m) 70s ago 35m 55.0M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 b35ee64d41ac 2024-05-20T23:05:59.125 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (2m) 70s ago 35m 54.6M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 ceee3eb06ecb 2024-05-20T23:05:59.125 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (2m) 70s ago 34m 46.2M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 e41094f18f45 2024-05-20T23:05:59.125 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (11m) 70s ago 34m 80.1M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:05:59.125 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (80s) 70s ago 32m 79.1M - 19.0.0-2695-g8a373da3 86045c3f4e75 b294685c019d 2024-05-20T23:05:59.125 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (74s) 70s ago 32m 77.4M - 19.0.0-2695-g8a373da3 86045c3f4e75 510f6b207ba0 2024-05-20T23:05:59.716 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:05:59 smithi055 ceph-mon[86116]: from='client.54912 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:59.717 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:05:59 smithi055 ceph-mon[88498]: from='client.54912 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:05:59.741 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions' 2024-05-20T23:06:00.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:05:59 smithi138 ceph-mon[67823]: from='client.54912 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:06:00.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:00 smithi055 ceph-mon[86116]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-05-20T23:06:00.983 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:00 smithi055 ceph-mon[88498]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-05-20T23:06:01.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:00 smithi138 ceph-mon[67823]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-05-20T23:06:01.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:01 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:01.900 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:06:01.901 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:06:01.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:06:01.902 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:06:01.902 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 15 2024-05-20T23:06:01.902 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:06:01.902 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:06:01.902 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:01 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:01.902 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:01 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:02.508 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-05-20T23:06:02.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:02 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2143579239' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:06:02.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:02 smithi055 ceph-mon[86116]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-20T23:06:02.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:02 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2143579239' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:06:02.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:02 smithi055 ceph-mon[88498]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-20T23:06:03.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:02 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2143579239' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:06:03.105 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:02 smithi138 ceph-mon[67823]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-20T23:06:04.220 INFO:teuthology.orchestra.run.smithi055.stdout:wait for servicemap items w/ changing names to refresh 2024-05-20T23:06:04.914 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'sleep 60' 2024-05-20T23:06:05.471 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:05 smithi055 ceph-mon[86116]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:05.471 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:05 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:05.471 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:05 smithi055 ceph-mon[88498]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:05.471 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:05 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:05.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:05 smithi138 ceph-mon[67823]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:05.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:05 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:07.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:07 smithi055 ceph-mon[86116]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-05-20T23:06:07.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:07 smithi055 ceph-mon[88498]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-05-20T23:06:07.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:06:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:06:07] "GET /metrics HTTP/1.1" 200 38410 "" "Prometheus/2.43.0" 2024-05-20T23:06:07.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:07 smithi138 ceph-mon[67823]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 538 B/s rd, 0 op/s 2024-05-20T23:06:09.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:09 smithi055 ceph-mon[86116]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-05-20T23:06:09.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:09 smithi055 ceph-mon[88498]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-05-20T23:06:09.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:09 smithi138 ceph-mon[67823]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-05-20T23:06:11.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:11 smithi055 ceph-mon[88498]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:11.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:11 smithi055 ceph-mon[86116]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:11.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:11 smithi138 ceph-mon[67823]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:12.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:12 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:12.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:12 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:12.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:12 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:13.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:13 smithi138 ceph-mon[67823]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:13.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:13 smithi055 ceph-mon[86116]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:13.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:13 smithi055 ceph-mon[88498]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:15.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:15 smithi138 ceph-mon[67823]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:15.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:15 smithi055 ceph-mon[86116]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:15.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:15 smithi055 ceph-mon[88498]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:17.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:17 smithi138 ceph-mon[67823]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:17.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:17 smithi055 ceph-mon[86116]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:17.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:17 smithi055 ceph-mon[88498]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:17.733 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:06:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:06:17] "GET /metrics HTTP/1.1" 200 38410 "" "Prometheus/2.43.0" 2024-05-20T23:06:19.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:19 smithi138 ceph-mon[67823]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:19.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:19 smithi055 ceph-mon[86116]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:19.734 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:19 smithi055 ceph-mon[88498]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:20.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:20.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:20.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:21.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:21 smithi138 ceph-mon[67823]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:21.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:21 smithi055 ceph-mon[86116]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:21.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:21 smithi055 ceph-mon[88498]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:22.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:22 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:22.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:22 smithi138 ceph-mon[67823]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:22.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:22 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:22.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:22 smithi055 ceph-mon[86116]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:22.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:22 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:22.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:22 smithi055 ceph-mon[88498]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:25.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:25 smithi055 ceph-mon[86116]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:25.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:25 smithi055 ceph-mon[88498]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:25.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:25 smithi138 ceph-mon[67823]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:27.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:27 smithi055 ceph-mon[86116]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:27.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:27 smithi055 ceph-mon[88498]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:27.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:06:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:06:27] "GET /metrics HTTP/1.1" 200 38414 "" "Prometheus/2.43.0" 2024-05-20T23:06:27.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:27 smithi138 ceph-mon[67823]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:29.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:29 smithi055 ceph-mon[88498]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:29.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:29 smithi055 ceph-mon[86116]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:29.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:29 smithi138 ceph-mon[67823]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:31.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:31 smithi055 ceph-mon[86116]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:31.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:31 smithi055 ceph-mon[88498]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:31.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:31 smithi138 ceph-mon[67823]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:32.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:32 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:32.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:32 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:32.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:32 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:33.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:33 smithi138 ceph-mon[67823]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:33.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:33 smithi055 ceph-mon[86116]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:33.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:33 smithi055 ceph-mon[88498]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:35.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:35 smithi138 ceph-mon[67823]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:06:35.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:35 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:35.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:35 smithi055 ceph-mon[86116]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:06:35.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:35 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:35.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:35 smithi055 ceph-mon[88498]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:06:35.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:35 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:37.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:37 smithi138 ceph-mon[67823]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:37.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:37 smithi055 ceph-mon[86116]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:37.732 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:06:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:06:37] "GET /metrics HTTP/1.1" 200 38408 "" "Prometheus/2.43.0" 2024-05-20T23:06:37.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:37 smithi055 ceph-mon[88498]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:39.580 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:39 smithi055 ceph-mon[86116]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:39.581 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:39 smithi055 ceph-mon[88498]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:39.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:39 smithi138 ceph-mon[67823]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:41 smithi138 ceph-mon[67823]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:06:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:06:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:06:41.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:41 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:06:41.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[86116]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:41.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:06:41.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:06:41.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:06:41.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:06:41.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[88498]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:41.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T23:06:41.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T23:06:41.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T23:06:41.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:41 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' 2024-05-20T23:06:42.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:42 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:42.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:42 smithi138 ceph-mon[67823]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:42.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:42 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:42.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:42 smithi055 ceph-mon[86116]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:42.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:42 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:42.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:42 smithi055 ceph-mon[88498]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:45.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:45 smithi055 ceph-mon[86116]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:06:45.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:45 smithi055 ceph-mon[88498]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:06:45.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:45 smithi138 ceph-mon[67823]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:06:47.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:47 smithi055 ceph-mon[86116]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:47.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:47 smithi055 ceph-mon[88498]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:47.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:06:47 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:06:47] "GET /metrics HTTP/1.1" 200 38408 "" "Prometheus/2.43.0" 2024-05-20T23:06:47.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:47 smithi138 ceph-mon[67823]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:49.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:49 smithi055 ceph-mon[86116]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:49.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:49 smithi055 ceph-mon[88498]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:49.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:49 smithi138 ceph-mon[67823]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:50.506 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:50 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:50.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:50 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:50.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:50 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:06:51.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:51 smithi055 ceph-mon[86116]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:51.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:51 smithi055 ceph-mon[88498]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:51.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:51 smithi138 ceph-mon[67823]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:52.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:52 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:52.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:52 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:52.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:52 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:06:53.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:53 smithi138 ceph-mon[67823]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:53.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:53 smithi055 ceph-mon[86116]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:53.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:53 smithi055 ceph-mon[88498]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:55.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:55 smithi138 ceph-mon[67823]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:55.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:55 smithi055 ceph-mon[86116]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:55.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:55 smithi055 ceph-mon[88498]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:56.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:56 smithi138 ceph-mon[67823]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:56.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:56 smithi055 ceph-mon[86116]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:56.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:56 smithi055 ceph-mon[88498]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:06:57.633 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:06:57 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:06:57] "GET /metrics HTTP/1.1" 200 38411 "" "Prometheus/2.43.0" 2024-05-20T23:06:59.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:06:59 smithi055 ceph-mon[86116]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:59.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:06:59 smithi055 ceph-mon[88498]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:06:59.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:06:59 smithi138 ceph-mon[67823]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:01.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:01 smithi055 ceph-mon[86116]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:01.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:01 smithi055 ceph-mon[88498]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:01.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:01 smithi138 ceph-mon[67823]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:02.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:02 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:02.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:02 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:02.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:02 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:03.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:03 smithi138 ceph-mon[67823]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:03.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:03 smithi055 ceph-mon[86116]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:03.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:03 smithi055 ceph-mon[88498]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:05.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:05 smithi138 ceph-mon[67823]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:05.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:05 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:07:05.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:05 smithi055 ceph-mon[86116]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:05.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:05 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:07:05.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:05 smithi055 ceph-mon[88498]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:05.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:05 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:07:07.220 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T23:07:07.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:07 smithi055 ceph-mon[86116]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:07:07.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:07 smithi055 ceph-mon[88498]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:07:07.482 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:07 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:07:07] "GET /metrics HTTP/1.1" 200 38411 "" "Prometheus/2.43.0" 2024-05-20T23:07:07.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:07 smithi138 ceph-mon[67823]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:07:09.246 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (12m) 2m ago 34m 26.2M - 0.25.0 c8568f914cd2 df8a6bb3b4f5 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi138 *:3000 running (11m) 2m ago 34m 81.3M - 9.4.12 2bacad6d85d8 c12c9c6bb205 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:iscsi.foo.smithi055.akoeuh smithi055 running (11m) 2m ago 33m 66.3M - 3.5 e1d6a67b021e 8c97436d2636 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi138 *:8443,9283,8765 running (13m) 2m ago 39m 455M - 19.0.0-2695-g8a373da3 86045c3f4e75 6ba160359cab 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (23m) 2m ago 40m 515M - 19.0.0-2695-g8a373da3 86045c3f4e75 8e45ecee526a 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (9m) 2m ago 40m 59.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 0581e8f15f4f 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi138 running (10m) 2m ago 39m 52.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 e6d39c7bef11 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (9m) 2m ago 39m 48.8M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 afab2b40c96b 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (13m) 2m ago 35m 15.7M - 1.5.0 0da6a335fe13 b04e9536a7e1 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi138 *:9100 running (13m) 2m ago 35m 15.0M - 1.5.0 0da6a335fe13 f2df46ba361b 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (7m) 2m ago 38m 60.9M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 ea50151bc837 2024-05-20T23:07:09.247 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (7m) 2m ago 38m 61.3M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 eb6928b01e1a 2024-05-20T23:07:09.248 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (6m) 2m ago 38m 56.1M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 632f1b435da2 2024-05-20T23:07:09.248 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (5m) 2m ago 37m 52.0M 2993M 19.0.0-2695-g8a373da3 86045c3f4e75 b7c4fb200e8e 2024-05-20T23:07:09.248 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi138 running (5m) 2m ago 37m 52.7M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 c32ea1e7b587 2024-05-20T23:07:09.248 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi138 running (4m) 2m ago 36m 55.0M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 b35ee64d41ac 2024-05-20T23:07:09.248 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi138 running (3m) 2m ago 36m 54.6M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 ceee3eb06ecb 2024-05-20T23:07:09.248 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi138 running (3m) 2m ago 36m 46.2M 3248M 19.0.0-2695-g8a373da3 86045c3f4e75 e41094f18f45 2024-05-20T23:07:09.248 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi138 *:9095 running (12m) 2m ago 35m 80.1M - 2.43.0 a07b618ecd1d c7fec3619ff9 2024-05-20T23:07:09.248 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi055.ymnqng smithi055 *:8000 running (2m) 2m ago 33m 79.1M - 19.0.0-2695-g8a373da3 86045c3f4e75 b294685c019d 2024-05-20T23:07:09.248 INFO:teuthology.orchestra.run.smithi055.stdout:rgw.foo.smithi138.sfpuja smithi138 *:8000 running (2m) 2m ago 33m 77.4M - 19.0.0-2695-g8a373da3 86045c3f4e75 510f6b207ba0 2024-05-20T23:07:09.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:09 smithi055 ceph-mon[86116]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:09.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:09 smithi055 ceph-mon[88498]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:09.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:09 smithi138 ceph-mon[67823]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:09.854 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions' 2024-05-20T23:07:10.458 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:10 smithi055 ceph-mon[86116]: from='client.54924 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:10.459 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:10 smithi055 ceph-mon[88498]: from='client.54924 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:10.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:10 smithi138 ceph-mon[67823]: from='client.54924 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:11.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:11 smithi055 ceph-mon[86116]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:07:11.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:11 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:11.422 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:11 smithi055 ceph-mon[88498]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:07:11.423 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:11 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:11.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:11 smithi138 ceph-mon[67823]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-20T23:07:11.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:11 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:11.917 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:07:11.917 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2024-05-20T23:07:11.917 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T23:07:11.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:07:11.917 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2024-05-20T23:07:11.917 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw": { 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 15 2024-05-20T23:07:11.918 INFO:teuthology.orchestra.run.smithi055.stdout: } 2024-05-20T23:07:11.919 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:07:12.538 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade status' 2024-05-20T23:07:12.563 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:12 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/3671752109' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:07:12.563 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:12 smithi055 ceph-mon[86116]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:12.564 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:12 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/3671752109' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:07:12.564 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:12 smithi055 ceph-mon[88498]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:12.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:12 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/3671752109' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:07:12.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:12 smithi138 ceph-mon[67823]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:14.427 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:07:14.427 INFO:teuthology.orchestra.run.smithi055.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T23:07:14.427 INFO:teuthology.orchestra.run.smithi055.stdout: "in_progress": true, 2024-05-20T23:07:14.427 INFO:teuthology.orchestra.run.smithi055.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout: "services_complete": [ 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw", 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr", 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout: "osd", 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout: "mon" 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout: "progress": "15/21 daemons upgraded", 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed.", 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout: "is_paused": true 2024-05-20T23:07:14.428 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:07:15.043 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T23:07:15.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:15 smithi055 ceph-mon[86116]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:15.401 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:15 smithi055 ceph-mon[88498]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:15.606 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:15 smithi138 ceph-mon[67823]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:16.427 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:16 smithi055 ceph-mon[86116]: from='client.54936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:16.427 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:16 smithi055 ceph-mon[88498]: from='client.54936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:16.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:16 smithi138 ceph-mon[67823]: from='client.54936 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:17.210 INFO:teuthology.orchestra.run.smithi055.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed. 2024-05-20T23:07:17.210 INFO:teuthology.orchestra.run.smithi055.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi055.akoeuh on host smithi055 failed. 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout: Upgrade daemon: iscsi.foo.smithi055.akoeuh: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi055.akoeuh ... 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout:Creating ceph-iscsi config... 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout:Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/iscsi-gateway.cfg 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout:Write file: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/iscsi.foo.smithi055.akoeuh/tcmu-runner-entrypoint.sh 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2ddd0bace8\x2d16f7\x2d11ef\x2dbc9a\x2dc7b262605968.slice/ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 1 from systemctl start ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout:Traceback (most recent call last): 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout: return _run_code(code, main_globals, None, 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-20T23:07:17.211 INFO:teuthology.orchestra.run.smithi055.stdout: exec(code, run_globals) 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5618, in 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 5606, in main 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3096, in command_deploy_from 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3131, in _common_deploy 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 3151, in _deploy_daemon_container 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1087, in deploy_daemon 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/__main__.py", line 1223, in deploy_daemon_units 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout: File "/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/cephadm.293581b3c9261c38becbd7affeef96028989e1871e1032006506e2d3cf7c5eeb/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout:RuntimeError: Failed command: systemctl start ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh: Job for ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service failed because the control process exited with error code. 2024-05-20T23:07:17.212 INFO:teuthology.orchestra.run.smithi055.stdout:See "systemctl status ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" and "journalctl -xeu ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@iscsi.foo.smithi055.akoeuh.service" for details. 2024-05-20T23:07:17.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:17 smithi055 ceph-mon[86116]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:17.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:17 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/2880801889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:07:17.483 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:17 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:07:17] "GET /metrics HTTP/1.1" 200 38411 "" "Prometheus/2.43.0" 2024-05-20T23:07:17.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:17 smithi055 ceph-mon[88498]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:17.483 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:17 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/2880801889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:07:17.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:17 smithi138 ceph-mon[67823]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:17.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:17 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/2880801889' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T23:07:17.776 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-05-20T23:07:19.306 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:19 smithi055 ceph-mon[86116]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:19.307 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:19 smithi055 ceph-mon[88498]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:19.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:19 smithi138 ceph-mon[67823]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:19.839 INFO:teuthology.orchestra.run.smithi055.stdout:true 2024-05-20T23:07:20.366 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:20 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:07:20.366 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:20 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/1417462938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:07:20.367 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:20 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:07:20.367 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:20 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/1417462938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:07:20.390 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-05-20T23:07:20.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:20 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:07:20.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:20 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/1417462938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:07:21.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:21 smithi138 ceph-mon[67823]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:21.616 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:21 smithi055 ceph-mon[86116]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:21.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:21 smithi055 ceph-mon[88498]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:22.406 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)" 2024-05-20T23:07:22.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:22 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:22.709 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:22 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:22.710 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:22 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:22.983 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-05-20T23:07:23.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:23 smithi138 ceph-mon[67823]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:23.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:23 smithi138 ceph-mon[67823]: from='client.? 172.21.15.55:0/4051659188' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:07:23.618 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:23 smithi055 ceph-mon[86116]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:23.618 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:23 smithi055 ceph-mon[86116]: from='client.? 172.21.15.55:0/4051659188' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:07:23.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:23 smithi055 ceph-mon[88498]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:23.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:23 smithi055 ceph-mon[88498]: from='client.? 172.21.15.55:0/4051659188' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T23:07:24.517 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:24 smithi055 ceph-mon[86116]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:24.517 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:24 smithi055 ceph-mon[88498]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:24.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:24 smithi138 ceph-mon[67823]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:24.994 INFO:teuthology.orchestra.run.smithi055.stdout:osd 8 2m ago - 2024-05-20T23:07:25.573 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:25 smithi055 ceph-mon[86116]: from='client.64885 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:25.573 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:25 smithi055 ceph-mon[88498]: from='client.64885 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:25.602 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T23:07:25.613 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi055.front.sepia.ceph.com 2024-05-20T23:07:25.613 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-05-20T23:07:25.614 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:25 smithi138 ceph-mon[67823]: from='client.64885 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:26.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:26 smithi055 ceph-mon[88498]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:26.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:26 smithi055 ceph-mon[86116]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:26.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:26 smithi138 ceph-mon[67823]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:27.366 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:27 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:07:27] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-05-20T23:07:28.363 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-05-20T23:07:28.363 INFO:teuthology.orchestra.run.smithi055.stdout: "image": "quay.io/ceph/ceph", 2024-05-20T23:07:28.363 INFO:teuthology.orchestra.run.smithi055.stdout: "registry": "quay.io", 2024-05-20T23:07:28.363 INFO:teuthology.orchestra.run.smithi055.stdout: "bare_image": "ceph/ceph", 2024-05-20T23:07:28.363 INFO:teuthology.orchestra.run.smithi055.stdout: "versions": [] 2024-05-20T23:07:28.363 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-05-20T23:07:29.008 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-05-20T23:07:29.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:29 smithi055 ceph-mon[86116]: from='client.54966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:29.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:29 smithi055 ceph-mon[86116]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:29.574 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:29 smithi055 ceph-mon[88498]: from='client.54966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:29.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:29 smithi055 ceph-mon[88498]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:29.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:29 smithi138 ceph-mon[67823]: from='client.54966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:29.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:29 smithi138 ceph-mon[67823]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:31.482 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:31 smithi055 ceph-mon[86116]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:31.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:31 smithi055 ceph-mon[88498]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:31.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:31 smithi138 ceph-mon[67823]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:31.745 INFO:teuthology.orchestra.run.smithi055.stdout: "16.2.0", 2024-05-20T23:07:32.295 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:32 smithi055 ceph-mon[86116]: from='client.54972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:32.295 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:32 smithi055 ceph-mon[86116]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:32.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:32 smithi055 ceph-mon[88498]: from='client.54972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:32.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:32 smithi055 ceph-mon[88498]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:32.326 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-05-20T23:07:32.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:32 smithi138 ceph-mon[67823]: from='client.54972 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:32.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:32 smithi138 ceph-mon[67823]: from='client.54849 -' entity='client.iscsi.foo.smithi055.akoeuh' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-20T23:07:33.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:33 smithi055 ceph-mon[86116]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:33.544 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:33 smithi055 ceph-mon[88498]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:33.605 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:33 smithi138 ceph-mon[67823]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-20T23:07:34.965 INFO:teuthology.orchestra.run.smithi055.stdout: "v16.2.2", 2024-05-20T23:07:34.965 INFO:teuthology.orchestra.run.smithi055.stdout: "v16.2.2-20210505", 2024-05-20T23:07:35.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:35 smithi055 ceph-mon[86116]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:35.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:35 smithi055 ceph-mon[86116]: from='client.54978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:35.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:35 smithi055 ceph-mon[86116]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:07:35.533 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:35 smithi055 ceph-mon[88498]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:35.533 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:35 smithi055 ceph-mon[88498]: from='client.54978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:35.533 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:35 smithi055 ceph-mon[88498]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:07:35.534 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-05-20T23:07:35.544 INFO:tasks.cephadm:Teardown begin 2024-05-20T23:07:35.544 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T23:07:35.576 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T23:07:35.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:35 smithi138 ceph-mon[67823]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-20T23:07:35.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:35 smithi138 ceph-mon[67823]: from='client.54978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T23:07:35.604 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:35 smithi138 ceph-mon[67823]: from='mgr.54108 172.21.15.55:0/1613532665' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T23:07:35.609 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-05-20T23:07:35.610 DEBUG:teuthology.orchestra.run.smithi055:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-20T23:07:35.634 DEBUG:teuthology.orchestra.run.smithi138:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-20T23:07:35.668 INFO:tasks.cephadm:Stopping all daemons... 2024-05-20T23:07:35.669 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-05-20T23:07:35.669 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.a 2024-05-20T23:07:35.972 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:35 smithi055 systemd[1]: Stopping Ceph mon.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:07:36.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:35 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a[86112]: 2024-05-20T23:07:35.980+0000 7f2ebf89b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:07:36.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:35 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a[86112]: 2024-05-20T23:07:35.980+0000 7f2ebf89b700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-05-20T23:07:36.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:36 smithi055 podman[129726]: 2024-05-20 23:07:36.083344494 +0000 UTC m=+0.198459792 container died 0581e8f15f4f3a909000b08bffebd230b0c3b65f4c2d78264f8e902265a9a323 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-05-20T23:07:36.232 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:35 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:23:07:35] ENGINE Bus STOPPING 2024-05-20T23:07:36.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:23:07:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-20T23:07:36.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:23:07:36] ENGINE Bus STOPPED 2024-05-20T23:07:36.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:23:07:36] ENGINE Bus STARTING 2024-05-20T23:07:36.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: CherryPy Checker: 2024-05-20T23:07:36.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: The Application mounted at '' has an empty config. 2024-05-20T23:07:36.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T23:07:36.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:23:07:36] ENGINE Serving on http://:::9283 2024-05-20T23:07:36.233 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:36 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: [20/May/2024:23:07:36] ENGINE Bus STARTED 2024-05-20T23:07:36.690 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:36 smithi055 podman[129726]: 2024-05-20 23:07:36.456336754 +0000 UTC m=+0.571452034 container cleanup 0581e8f15f4f3a909000b08bffebd230b0c3b65f4c2d78264f8e902265a9a323 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-05-20T23:07:36.690 INFO:journalctl@ceph.mon.a.smithi055.stdout:May 20 23:07:36 smithi055 bash[129726]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-a 2024-05-20T23:07:36.827 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.a.service' 2024-05-20T23:07:36.877 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:07:36.877 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-05-20T23:07:36.877 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-05-20T23:07:36.878 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.c 2024-05-20T23:07:36.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:36 smithi055 systemd[1]: Stopping Ceph mon.c for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:07:37.482 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: ::ffff:172.21.15.138 - - [20/May/2024:23:07:37] "GET /metrics HTTP/1.1" 200 38403 "" "Prometheus/2.43.0" 2024-05-20T23:07:37.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c[88494]: 2024-05-20T23:07:37.176+0000 7f28822f3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:07:37.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:37 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c[88494]: 2024-05-20T23:07:37.176+0000 7f28822f3700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-05-20T23:07:37.482 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:37 smithi055 podman[129858]: 2024-05-20 23:07:37.413478501 +0000 UTC m=+0.333246346 container died afab2b40c96b24dfed361f4aa4652c123e61274d8eab14e01d8e0aa248ea7af4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:07:37.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:37 smithi055 podman[129858]: 2024-05-20 23:07:37.752409762 +0000 UTC m=+0.672177600 container cleanup afab2b40c96b24dfed361f4aa4652c123e61274d8eab14e01d8e0aa248ea7af4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240514) 2024-05-20T23:07:37.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:37 smithi055 bash[129858]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c 2024-05-20T23:07:38.180 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.c.service' 2024-05-20T23:07:38.203 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:37 smithi055 podman[129872]: 2024-05-20 23:07:37.945626287 +0000 UTC m=+0.534342518 container remove afab2b40c96b24dfed361f4aa4652c123e61274d8eab14e01d8e0aa248ea7af4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-c, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-05-20T23:07:38.203 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:38 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.c.service: Deactivated successfully. 2024-05-20T23:07:38.203 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:38 smithi055 systemd[1]: Stopped Ceph mon.c for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:07:38.203 INFO:journalctl@ceph.mon.c.smithi055.stdout:May 20 23:07:38 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.c.service: Consumed 6.445s CPU time. 2024-05-20T23:07:38.217 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:07:38.217 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-05-20T23:07:38.217 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-05-20T23:07:38.217 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.b 2024-05-20T23:07:38.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:38 smithi138 systemd[1]: Stopping Ceph mon.b for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:07:38.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b[67819]: 2024-05-20T23:07:38.562+0000 7f9e4addf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:07:38.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b[67819]: 2024-05-20T23:07:38.562+0000 7f9e4addf700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-05-20T23:07:38.855 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:38 smithi138 podman[100747]: 2024-05-20 23:07:38.665394252 +0000 UTC m=+0.231413216 container died e6d39c7bef115831bb5e8b8b6adcffb0cebffdd7dfb35ae08d7f778e5dc79a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:07:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:39 smithi138 podman[100747]: 2024-05-20 23:07:39.130463987 +0000 UTC m=+0.696482950 container cleanup e6d39c7bef115831bb5e8b8b6adcffb0cebffdd7dfb35ae08d7f778e5dc79a10 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:07:39.355 INFO:journalctl@ceph.mon.b.smithi138.stdout:May 20 23:07:39 smithi138 bash[100747]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mon-b 2024-05-20T23:07:39.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mon.b.service' 2024-05-20T23:07:39.622 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:07:39.623 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-05-20T23:07:39.623 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-05-20T23:07:39.623 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.y 2024-05-20T23:07:39.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:39 smithi055 systemd[1]: Stopping Ceph mgr.y for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:07:40.232 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:39 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T23:07:39.939+0000 7f06abd51700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:07:40.232 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:39 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y[63091]: 2024-05-20T23:07:39.939+0000 7f06abd51700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-05-20T23:07:40.232 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:39 smithi055 podman[129979]: 2024-05-20 23:07:39.972169753 +0000 UTC m=+0.145700683 container died 8e45ecee526afeda5089451cd01774902c67577e6e44eff8cf37b01e60146b25 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-05-20T23:07:40.608 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:40 smithi055 podman[129979]: 2024-05-20 23:07:40.366334207 +0000 UTC m=+0.539865136 container cleanup 8e45ecee526afeda5089451cd01774902c67577e6e44eff8cf37b01e60146b25 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-05-20T23:07:40.608 INFO:journalctl@ceph.mgr.y.smithi055.stdout:May 20 23:07:40 smithi055 bash[129979]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-y 2024-05-20T23:07:40.791 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.y.service' 2024-05-20T23:07:40.827 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:07:40.827 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-05-20T23:07:40.828 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-05-20T23:07:40.828 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.x 2024-05-20T23:07:41.105 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 23:07:40 smithi138 systemd[1]: Stopping Ceph mgr.x for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:07:41.459 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 23:07:41 smithi138 podman[100871]: 2024-05-20 23:07:41.184883214 +0000 UTC m=+0.147000168 container died 6ba160359cab3eae8a55d4e2191214ff32a05ef036e887d933d0e14c846ad80f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-05-20T23:07:41.773 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 23:07:41 smithi138 podman[100871]: 2024-05-20 23:07:41.469411888 +0000 UTC m=+0.431528839 container cleanup 6ba160359cab3eae8a55d4e2191214ff32a05ef036e887d933d0e14c846ad80f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-05-20T23:07:41.773 INFO:journalctl@ceph.mgr.x.smithi138.stdout:May 20 23:07:41 smithi138 bash[100871]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-mgr-x 2024-05-20T23:07:41.943 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@mgr.x.service' 2024-05-20T23:07:41.977 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:07:41.978 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-05-20T23:07:41.978 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-05-20T23:07:41.978 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.0 2024-05-20T23:07:42.278 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:42 smithi055 systemd[1]: Stopping Ceph osd.0 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:07:42.732 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[95806]: 2024-05-20T23:07:42.286+0000 7f806eac7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:07:42.732 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[95806]: 2024-05-20T23:07:42.286+0000 7f806eac7700 -1 osd.0 161 *** Got signal Terminated *** 2024-05-20T23:07:42.732 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:42 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0[95806]: 2024-05-20T23:07:42.286+0000 7f806eac7700 -1 osd.0 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:07:43.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:07:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:07:43.550Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-05-20T23:07:43.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:07:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:07:43.550Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:07:47.904 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:47 smithi055 podman[130100]: 2024-05-20 23:07:47.640906556 +0000 UTC m=+5.457172704 container died ea50151bc837e0f0c72aca18bfe06aa974c1b496321694942faf9b39f38e1ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0) 2024-05-20T23:07:48.232 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:48 smithi055 podman[130100]: 2024-05-20 23:07:48.003685721 +0000 UTC m=+5.819951871 container cleanup ea50151bc837e0f0c72aca18bfe06aa974c1b496321694942faf9b39f38e1ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-05-20T23:07:48.232 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:48 smithi055 bash[130100]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0 2024-05-20T23:07:48.643 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:48 smithi055 podman[130117]: 2024-05-20 23:07:48.297445841 +0000 UTC m=+0.646210809 container remove ea50151bc837e0f0c72aca18bfe06aa974c1b496321694942faf9b39f38e1ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240514) 2024-05-20T23:07:48.941 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:48 smithi055 podman[130204]: 2024-05-20 23:07:48.555613719 +0000 UTC m=+0.018236600 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:07:48.941 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:48 smithi055 podman[130204]: 2024-05-20 23:07:48.681665111 +0000 UTC m=+0.144288028 container create aecc4a91d19d6810cb26d7db836708a72c9e5a8fec4e498cd8a4f9bddbea2191 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-05-20T23:07:48.941 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:48 smithi055 podman[130204]: 2024-05-20 23:07:48.875839001 +0000 UTC m=+0.338461876 container init aecc4a91d19d6810cb26d7db836708a72c9e5a8fec4e498cd8a4f9bddbea2191 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-05-20T23:07:48.941 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:48 smithi055 podman[130204]: 2024-05-20 23:07:48.879197393 +0000 UTC m=+0.341820277 container start aecc4a91d19d6810cb26d7db836708a72c9e5a8fec4e498cd8a4f9bddbea2191 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-20T23:07:48.942 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:48 smithi055 podman[130204]: 2024-05-20 23:07:48.950620679 +0000 UTC m=+0.413243562 container attach aecc4a91d19d6810cb26d7db836708a72c9e5a8fec4e498cd8a4f9bddbea2191 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-20T23:07:49.209 INFO:journalctl@ceph.osd.0.smithi055.stdout:May 20 23:07:49 smithi055 podman[130204]: 2024-05-20 23:07:49.191305154 +0000 UTC m=+0.653928028 container died aecc4a91d19d6810cb26d7db836708a72c9e5a8fec4e498cd8a4f9bddbea2191 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:07:49.824 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.0.service' 2024-05-20T23:07:49.866 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:07:49.867 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-05-20T23:07:49.867 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-05-20T23:07:49.867 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.1 2024-05-20T23:07:50.232 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:49 smithi055 systemd[1]: Stopping Ceph osd.1 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:07:50.732 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[100957]: 2024-05-20T23:07:50.285+0000 7fde1a3e7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:07:50.732 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[100957]: 2024-05-20T23:07:50.285+0000 7fde1a3e7700 -1 osd.1 161 *** Got signal Terminated *** 2024-05-20T23:07:50.732 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1[100957]: 2024-05-20T23:07:50.285+0000 7fde1a3e7700 -1 osd.1 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:07:53.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:07:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:07:53.550Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:07:53.983 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:07:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:07:53.551Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-05-20T23:07:53.983 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:07:53 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:07:53.551Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:07:56.050 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:55 smithi055 podman[130318]: 2024-05-20 23:07:55.743618283 +0000 UTC m=+5.599409785 container stop eb6928b01e1a6c2af76c263690046b5f8f98b06e7e501358991787da1a18429d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-05-20T23:07:56.050 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:55 smithi055 podman[130318]: 2024-05-20 23:07:55.769842448 +0000 UTC m=+5.625633950 container died eb6928b01e1a6c2af76c263690046b5f8f98b06e7e501358991787da1a18429d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-05-20T23:07:56.482 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:56 smithi055 podman[130318]: 2024-05-20 23:07:56.133307986 +0000 UTC m=+5.989099482 container cleanup eb6928b01e1a6c2af76c263690046b5f8f98b06e7e501358991787da1a18429d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-05-20T23:07:56.482 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:56 smithi055 bash[130318]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1 2024-05-20T23:07:56.482 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:56 smithi055 podman[130337]: 2024-05-20 23:07:56.309454339 +0000 UTC m=+0.553929354 container remove eb6928b01e1a6c2af76c263690046b5f8f98b06e7e501358991787da1a18429d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux ) 2024-05-20T23:07:57.062 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:56 smithi055 podman[130423]: 2024-05-20 23:07:56.677710725 +0000 UTC m=+0.017477116 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:07:57.062 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:56 smithi055 podman[130423]: 2024-05-20 23:07:56.777137566 +0000 UTC m=+0.116903936 container create 04b78b583bd7b2c2012b45efe1aad66f559f0a30633cf16427b7684d5ed25f04 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T23:07:57.063 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:56 smithi055 podman[130423]: 2024-05-20 23:07:56.996707159 +0000 UTC m=+0.336473530 container init 04b78b583bd7b2c2012b45efe1aad66f559f0a30633cf16427b7684d5ed25f04 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True) 2024-05-20T23:07:57.063 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:57 smithi055 podman[130423]: 2024-05-20 23:07:57.000075827 +0000 UTC m=+0.339842211 container start 04b78b583bd7b2c2012b45efe1aad66f559f0a30633cf16427b7684d5ed25f04 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-05-20T23:07:57.063 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:57 smithi055 podman[130423]: 2024-05-20 23:07:57.071446181 +0000 UTC m=+0.411212561 container attach 04b78b583bd7b2c2012b45efe1aad66f559f0a30633cf16427b7684d5ed25f04 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:07:57.327 INFO:journalctl@ceph.osd.1.smithi055.stdout:May 20 23:07:57 smithi055 podman[130423]: 2024-05-20 23:07:57.31661216 +0000 UTC m=+0.656378543 container died 04b78b583bd7b2c2012b45efe1aad66f559f0a30633cf16427b7684d5ed25f04 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-20T23:07:57.885 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.1.service' 2024-05-20T23:07:57.929 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:07:57.930 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-05-20T23:07:57.930 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-05-20T23:07:57.930 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.2 2024-05-20T23:07:58.228 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:07:57 smithi055 systemd[1]: Stopping Ceph osd.2 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:07:58.482 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:07:58 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[107091]: 2024-05-20T23:07:58.236+0000 7f55e601a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:07:58.482 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:07:58 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[107091]: 2024-05-20T23:07:58.236+0000 7f55e601a700 -1 osd.2 161 *** Got signal Terminated *** 2024-05-20T23:07:58.482 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:07:58 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2[107091]: 2024-05-20T23:07:58.236+0000 7f55e601a700 -1 osd.2 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:08:03.970 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:03.551Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:08:03.970 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:03.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-05-20T23:08:03.970 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:03 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:03.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:08:03.971 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:03 smithi055 podman[130532]: 2024-05-20 23:08:03.72679398 +0000 UTC m=+5.588789464 container died 632f1b435da2e0fec3f451bec9454fba4a31f8b2fbbd9f7383814420d5e55ab5 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T23:08:04.232 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:04 smithi055 podman[130532]: 2024-05-20 23:08:04.036796787 +0000 UTC m=+5.898792256 container cleanup 632f1b435da2e0fec3f451bec9454fba4a31f8b2fbbd9f7383814420d5e55ab5 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514) 2024-05-20T23:08:04.232 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:04 smithi055 bash[130532]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2 2024-05-20T23:08:04.710 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:04 smithi055 podman[130548]: 2024-05-20 23:08:04.271368824 +0000 UTC m=+0.540367576 container remove 632f1b435da2e0fec3f451bec9454fba4a31f8b2fbbd9f7383814420d5e55ab5 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514) 2024-05-20T23:08:04.982 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:04 smithi055 podman[130634]: 2024-05-20 23:08:04.622660284 +0000 UTC m=+0.017881175 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:08:04.982 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:04 smithi055 podman[130634]: 2024-05-20 23:08:04.738977683 +0000 UTC m=+0.134198570 container create aa3e957e12615be8b364fa3e4746f7b59e0345e765a0f296985a071df293f4ea (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:08:04.982 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:04 smithi055 podman[130634]: 2024-05-20 23:08:04.941791205 +0000 UTC m=+0.337012092 container init aa3e957e12615be8b364fa3e4746f7b59e0345e765a0f296985a071df293f4ea (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:08:04.983 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:04 smithi055 podman[130634]: 2024-05-20 23:08:04.945431077 +0000 UTC m=+0.340651969 container start aa3e957e12615be8b364fa3e4746f7b59e0345e765a0f296985a071df293f4ea (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:08:05.482 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:05 smithi055 podman[130634]: 2024-05-20 23:08:05.041443822 +0000 UTC m=+0.436664715 container attach aa3e957e12615be8b364fa3e4746f7b59e0345e765a0f296985a071df293f4ea (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-05-20T23:08:05.482 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:05 smithi055 podman[130634]: 2024-05-20 23:08:05.252568815 +0000 UTC m=+0.647789701 container died aa3e957e12615be8b364fa3e4746f7b59e0345e765a0f296985a071df293f4ea (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-05-20T23:08:05.781 INFO:journalctl@ceph.osd.2.smithi055.stdout:May 20 23:08:05 smithi055 podman[130660]: 2024-05-20 23:08:05.778479226 +0000 UTC m=+0.518883218 container remove aa3e957e12615be8b364fa3e4746f7b59e0345e765a0f296985a071df293f4ea (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:08:05.787 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.2.service' 2024-05-20T23:08:05.833 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:08:05.834 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-05-20T23:08:05.834 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-05-20T23:08:05.834 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.3 2024-05-20T23:08:06.139 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:05 smithi055 systemd[1]: Stopping Ceph osd.3 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:08:06.482 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:06 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[113717]: 2024-05-20T23:08:06.148+0000 7feb19fd0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:08:06.482 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:06 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[113717]: 2024-05-20T23:08:06.148+0000 7feb19fd0700 -1 osd.3 161 *** Got signal Terminated *** 2024-05-20T23:08:06.482 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:06 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3[113717]: 2024-05-20T23:08:06.148+0000 7feb19fd0700 -1 osd.3 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:08:09.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:08 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:08.790+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:09.793 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:09 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:09.473+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:10.104 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:09 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:09.802+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:10.794 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:10 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:10.453+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:11.104 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:10 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:10.804+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:11.797 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:11 smithi055 podman[130740]: 2024-05-20 23:08:11.554200603 +0000 UTC m=+5.516101207 container died b7c4fb200e8e8a989c46586eb4f1dd06f6f55e09ef9a037acd1dc64c7a73ca42 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-05-20T23:08:11.805 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:11.389+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:11.806 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:11.472+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:12.064 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:11 smithi055 podman[130740]: 2024-05-20 23:08:11.897527343 +0000 UTC m=+5.859427953 container cleanup b7c4fb200e8e8a989c46586eb4f1dd06f6f55e09ef9a037acd1dc64c7a73ca42 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T23:08:12.064 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:11 smithi055 bash[130740]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3 2024-05-20T23:08:12.104 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:11 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:11.814+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:12.482 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:12 smithi055 podman[130759]: 2024-05-20 23:08:12.073827917 +0000 UTC m=+0.515437598 container remove b7c4fb200e8e8a989c46586eb4f1dd06f6f55e09ef9a037acd1dc64c7a73ca42 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-20T23:08:12.605 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:12.490+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:12.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:12.315+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:12.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:12.339+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:12.775 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:12 smithi055 podman[130848]: 2024-05-20 23:08:12.413502044 +0000 UTC m=+0.017922546 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:08:12.776 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:12 smithi055 podman[130848]: 2024-05-20 23:08:12.533044464 +0000 UTC m=+0.137465008 container create c343d643b6d621344cc3705d3cf00985b6035ed1a2fa9d90a84c6a8ed29a26b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-20T23:08:12.776 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:12 smithi055 podman[130848]: 2024-05-20 23:08:12.726912824 +0000 UTC m=+0.331333327 container init c343d643b6d621344cc3705d3cf00985b6035ed1a2fa9d90a84c6a8ed29a26b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-20T23:08:12.776 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:12 smithi055 podman[130848]: 2024-05-20 23:08:12.730335198 +0000 UTC m=+0.334755710 container start c343d643b6d621344cc3705d3cf00985b6035ed1a2fa9d90a84c6a8ed29a26b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-20T23:08:12.776 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:12 smithi055 podman[130848]: 2024-05-20 23:08:12.785227554 +0000 UTC m=+0.389648081 container attach c343d643b6d621344cc3705d3cf00985b6035ed1a2fa9d90a84c6a8ed29a26b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-20T23:08:13.104 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:12 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:12.774+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:13.234 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:13 smithi055 podman[130848]: 2024-05-20 23:08:13.032143542 +0000 UTC m=+0.636564045 container died c343d643b6d621344cc3705d3cf00985b6035ed1a2fa9d90a84c6a8ed29a26b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-20T23:08:13.582 INFO:journalctl@ceph.osd.3.smithi055.stdout:May 20 23:08:13 smithi055 podman[130874]: 2024-05-20 23:08:13.572194727 +0000 UTC m=+0.534104620 container remove c343d643b6d621344cc3705d3cf00985b6035ed1a2fa9d90a84c6a8ed29a26b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=) 2024-05-20T23:08:13.582 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:13.552Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-05-20T23:08:13.582 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:13.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-05-20T23:08:13.582 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:13 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:13.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:08:13.587 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.3.service' 2024-05-20T23:08:13.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:13.303+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:13.605 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:13.477+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:13.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:13.366+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:13.628 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:08:13.629 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-05-20T23:08:13.629 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-05-20T23:08:13.629 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.4 2024-05-20T23:08:13.939 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:13 smithi138 systemd[1]: Stopping Ceph osd.4 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:08:13.939 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:13.731+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:14.319 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:13.948+0000 7f6e0f73d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:08:14.320 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:13.948+0000 7f6e0f73d700 -1 osd.4 161 *** Got signal Terminated *** 2024-05-20T23:08:14.320 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:13 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:13.948+0000 7f6e0f73d700 -1 osd.4 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:08:14.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:14.329+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:14.605 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:14.429+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:14.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:14.415+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:15.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:14 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:14.732+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:15.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:15.319+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:15.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:15.406+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:15.606 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:15.468+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:16.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:15 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:15.765+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:16.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:16.329+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:16.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:16.431+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:16.605 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:16.448+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:17.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:16.744+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:17.605 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:17.489+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:17.605 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:17.489+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.754675+0000 front 2024-05-20T23:07:52.754598+0000 (oldest deadline 2024-05-20T23:08:17.454667+0000) 2024-05-20T23:08:17.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:17.342+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:17.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:17.342+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:17.606 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:17.403+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:17.606 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:17.403+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:18.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:17 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:17.737+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:18.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:18.340+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:18.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:18.340+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:18.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:18.448+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:18.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:18.448+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:18.606 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:18.441+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:43.552481+0000 front 2024-05-20T23:07:43.552590+0000 (oldest deadline 2024-05-20T23:08:09.452442+0000) 2024-05-20T23:08:18.606 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4[78317]: 2024-05-20T23:08:18.441+0000 7f6e0bd31700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.754675+0000 front 2024-05-20T23:07:52.754598+0000 (oldest deadline 2024-05-20T23:08:17.454667+0000) 2024-05-20T23:08:19.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:18.742+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:19.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:18 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:18.742+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:54.424717+0000 front 2024-05-20T23:07:54.424589+0000 (oldest deadline 2024-05-20T23:08:18.524514+0000) 2024-05-20T23:08:19.605 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:19 smithi138 podman[101008]: 2024-05-20 23:08:19.386743796 +0000 UTC m=+5.544336932 container died c32ea1e7b5872a5188932294f22b5cd99a0a3e05f4bc4d09ad6b612c0f02cbf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-05-20T23:08:19.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:19.312+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:19.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:19.312+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:19.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:19.407+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:19.606 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:19.407+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:20.105 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:19 smithi138 podman[101008]: 2024-05-20 23:08:19.856985244 +0000 UTC m=+6.014578389 container cleanup c32ea1e7b5872a5188932294f22b5cd99a0a3e05f4bc4d09ad6b612c0f02cbf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4, ceph=True, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-20T23:08:20.105 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:19 smithi138 bash[101008]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4 2024-05-20T23:08:20.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:19.741+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:20.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:19 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:19.741+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:54.424717+0000 front 2024-05-20T23:07:54.424589+0000 (oldest deadline 2024-05-20T23:08:18.524514+0000) 2024-05-20T23:08:20.390 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:20 smithi138 podman[101023]: 2024-05-20 23:08:20.14220446 +0000 UTC m=+0.750303506 container remove c32ea1e7b5872a5188932294f22b5cd99a0a3e05f4bc4d09ad6b612c0f02cbf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-05-20T23:08:20.391 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:20 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:20.329+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:20.391 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:20 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:20.329+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:20.690 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:20 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:20.398+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:20.690 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:20 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:20.398+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:20.690 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:20 smithi138 podman[101115]: 2024-05-20 23:08:20.378259294 +0000 UTC m=+0.017296411 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:08:20.691 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:20 smithi138 podman[101115]: 2024-05-20 23:08:20.510818834 +0000 UTC m=+0.149855963 container create ab616d3fb9169d55b91fb11ecbff8282c9f67a3fb5cd894650a138719b6dcc6b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-20T23:08:21.015 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:20 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:20.697+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:21.015 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:20 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:20.697+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:54.424717+0000 front 2024-05-20T23:07:54.424589+0000 (oldest deadline 2024-05-20T23:08:18.524514+0000) 2024-05-20T23:08:21.015 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:20 smithi138 podman[101115]: 2024-05-20 23:08:20.73009313 +0000 UTC m=+0.369130249 container init ab616d3fb9169d55b91fb11ecbff8282c9f67a3fb5cd894650a138719b6dcc6b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5) 2024-05-20T23:08:21.015 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:20 smithi138 podman[101115]: 2024-05-20 23:08:20.733380838 +0000 UTC m=+0.372417952 container start ab616d3fb9169d55b91fb11ecbff8282c9f67a3fb5cd894650a138719b6dcc6b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240514, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:08:21.015 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:20 smithi138 podman[101115]: 2024-05-20 23:08:20.805146569 +0000 UTC m=+0.444183688 container attach ab616d3fb9169d55b91fb11ecbff8282c9f67a3fb5cd894650a138719b6dcc6b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux ) 2024-05-20T23:08:21.287 INFO:journalctl@ceph.osd.4.smithi138.stdout:May 20 23:08:21 smithi138 podman[101115]: 2024-05-20 23:08:21.024889077 +0000 UTC m=+0.663926205 container died ab616d3fb9169d55b91fb11ecbff8282c9f67a3fb5cd894650a138719b6dcc6b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:08:21.604 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:21.308+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:21.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:21.308+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:21.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:21.444+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:21.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:21.444+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:21.605 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:21.297Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-05-20T23:08:21.605 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:21.299Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-05-20T23:08:21.605 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:21.300Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-05-20T23:08:21.606 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:21.302Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-05-20T23:08:21.606 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:21.302Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-05-20T23:08:21.669 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.4.service' 2024-05-20T23:08:21.708 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:08:21.708 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-05-20T23:08:21.708 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-05-20T23:08:21.709 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.5 2024-05-20T23:08:22.030 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:21.728+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:22.030 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:21 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:21.728+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:54.424717+0000 front 2024-05-20T23:07:54.424589+0000 (oldest deadline 2024-05-20T23:08:18.524514+0000) 2024-05-20T23:08:22.030 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:21 smithi138 systemd[1]: Stopping Ceph osd.5 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:08:22.030 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:22.038+0000 7f46dea28700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:08:22.030 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:22.038+0000 7f46dea28700 -1 osd.5 161 *** Got signal Terminated *** 2024-05-20T23:08:22.030 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:22.038+0000 7f46dea28700 -1 osd.5 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:08:22.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:22.418+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:22.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:22.418+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:22.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:22.339+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:22.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:22.339+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:23.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:22.736+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:23.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:22 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:22.736+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:54.424717+0000 front 2024-05-20T23:07:54.424589+0000 (oldest deadline 2024-05-20T23:08:18.524514+0000) 2024-05-20T23:08:23.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:23.453+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:23.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:23.453+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:23.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:23.361+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:23.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:23.361+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:23.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:23.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:08:23.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:23.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-05-20T23:08:23.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:23 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:23.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:08:24.106 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:23.779+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:24.106 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:23 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:23.779+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:54.424717+0000 front 2024-05-20T23:07:54.424589+0000 (oldest deadline 2024-05-20T23:08:18.524514+0000) 2024-05-20T23:08:24.796 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:24.381+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:24.797 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:24.381+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:24.797 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:24.451+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:24.797 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:24.451+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:25.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:24.805+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:25.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:24 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:24.805+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:54.424717+0000 front 2024-05-20T23:07:54.424589+0000 (oldest deadline 2024-05-20T23:08:18.524514+0000) 2024-05-20T23:08:25.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:25.467+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:25.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:25.467+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:25.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:25.352+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:25.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:25.352+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:26.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:25.770+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:26.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:25 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:25.770+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:54.424717+0000 front 2024-05-20T23:07:54.424589+0000 (oldest deadline 2024-05-20T23:08:18.524514+0000) 2024-05-20T23:08:26.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:26 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:26.516+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:26.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:26 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:26.516+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:26.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:26 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:26.304+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:26.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:26 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:26.304+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:26.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:26 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:26.304+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:00.666526+0000 front 2024-05-20T23:08:00.666642+0000 (oldest deadline 2024-05-20T23:08:25.366363+0000) 2024-05-20T23:08:27.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:26 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:26.752+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:47.023104+0000 front 2024-05-20T23:07:47.023049+0000 (oldest deadline 2024-05-20T23:08:08.122872+0000) 2024-05-20T23:08:27.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:26 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5[82787]: 2024-05-20T23:08:26.752+0000 7f46d7014700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:54.424717+0000 front 2024-05-20T23:07:54.424589+0000 (oldest deadline 2024-05-20T23:08:18.524514+0000) 2024-05-20T23:08:27.605 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:27 smithi138 podman[101220]: 2024-05-20 23:08:27.43187828 +0000 UTC m=+5.486822960 container died b35ee64d41acae8baac9cc241f1c7cd95ad689e3d1182c0c2fe81c033fe08ead (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-20T23:08:27.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:27 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:27.332+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:27.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:27 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:27.332+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:27.606 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:27 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:27.332+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:00.666526+0000 front 2024-05-20T23:08:00.666642+0000 (oldest deadline 2024-05-20T23:08:25.366363+0000) 2024-05-20T23:08:27.606 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:27 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:27.515+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:27.606 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:27 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:27.515+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:27.606 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:27 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:27.515+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:28.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:27 smithi138 podman[101220]: 2024-05-20 23:08:27.902150281 +0000 UTC m=+5.957094942 container cleanup b35ee64d41acae8baac9cc241f1c7cd95ad689e3d1182c0c2fe81c033fe08ead (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, ceph=True) 2024-05-20T23:08:28.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:27 smithi138 bash[101220]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5 2024-05-20T23:08:28.472 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:28 smithi138 podman[101235]: 2024-05-20 23:08:28.223869549 +0000 UTC m=+0.789033475 container remove b35ee64d41acae8baac9cc241f1c7cd95ad689e3d1182c0c2fe81c033fe08ead (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux ) 2024-05-20T23:08:28.472 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:28.344+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:28.473 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:28.344+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:28.473 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:28.344+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:00.666526+0000 front 2024-05-20T23:08:00.666642+0000 (oldest deadline 2024-05-20T23:08:25.366363+0000) 2024-05-20T23:08:28.818 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:28.481+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:28.818 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:28.481+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:28.818 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:28 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:28.481+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:28.819 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:28 smithi138 podman[101325]: 2024-05-20 23:08:28.575362501 +0000 UTC m=+0.108907827 container create 2ca0117517bce6cbd410500ee7cf9f1f48506442c5cb7ee9ec918bde725c564d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240514) 2024-05-20T23:08:28.819 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:28 smithi138 podman[101325]: 2024-05-20 23:08:28.48429302 +0000 UTC m=+0.017838323 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:08:28.819 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:28 smithi138 podman[101325]: 2024-05-20 23:08:28.827973791 +0000 UTC m=+0.361519067 container init 2ca0117517bce6cbd410500ee7cf9f1f48506442c5cb7ee9ec918bde725c564d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T23:08:29.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:28 smithi138 podman[101325]: 2024-05-20 23:08:28.831561783 +0000 UTC m=+0.365107075 container start 2ca0117517bce6cbd410500ee7cf9f1f48506442c5cb7ee9ec918bde725c564d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-05-20T23:08:29.105 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:28 smithi138 podman[101325]: 2024-05-20 23:08:28.911470751 +0000 UTC m=+0.445016037 container attach 2ca0117517bce6cbd410500ee7cf9f1f48506442c5cb7ee9ec918bde725c564d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-20T23:08:29.431 INFO:journalctl@ceph.osd.5.smithi138.stdout:May 20 23:08:29 smithi138 podman[101325]: 2024-05-20 23:08:29.124795689 +0000 UTC m=+0.658340975 container died 2ca0117517bce6cbd410500ee7cf9f1f48506442c5cb7ee9ec918bde725c564d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-5-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-05-20T23:08:29.431 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:29.338+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:29.432 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:29.338+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:29.432 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:29.338+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:00.666526+0000 front 2024-05-20T23:08:00.666642+0000 (oldest deadline 2024-05-20T23:08:25.366363+0000) 2024-05-20T23:08:29.706 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:29.519+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:29.706 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:29.519+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:29.706 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:29 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:29.519+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:29.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.5.service' 2024-05-20T23:08:29.771 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:08:29.771 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-05-20T23:08:29.771 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-05-20T23:08:29.771 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.6 2024-05-20T23:08:30.084 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:29 smithi138 systemd[1]: Stopping Ceph osd.6 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:08:30.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:30.093+0000 7f82be4ca700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:08:30.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:30.093+0000 7f82be4ca700 -1 osd.6 161 *** Got signal Terminated *** 2024-05-20T23:08:30.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:30.093+0000 7f82be4ca700 -1 osd.6 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:08:30.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:30.291+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:30.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:30.291+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:30.355 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:30.291+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:00.666526+0000 front 2024-05-20T23:08:00.666642+0000 (oldest deadline 2024-05-20T23:08:25.366363+0000) 2024-05-20T23:08:30.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:30.490+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:30.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:30.490+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:30.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:30 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:30.490+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:31.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:31.507+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:31.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:31.507+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:31.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:31.507+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:31.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:31.307+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:31.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:31.307+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:31.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:31 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:31.307+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:00.666526+0000 front 2024-05-20T23:08:00.666642+0000 (oldest deadline 2024-05-20T23:08:25.366363+0000) 2024-05-20T23:08:32.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:32.537+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:32.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:32.537+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:32.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:32.537+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:32.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:32.292+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:32.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:32.292+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:32.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:32 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:32.292+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:00.666526+0000 front 2024-05-20T23:08:00.666642+0000 (oldest deadline 2024-05-20T23:08:25.366363+0000) 2024-05-20T23:08:33.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:33.275+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:33.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:33.275+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:33.605 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:33.275+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:00.666526+0000 front 2024-05-20T23:08:00.666642+0000 (oldest deadline 2024-05-20T23:08:25.366363+0000) 2024-05-20T23:08:33.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:33.512+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:33.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:33.512+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:33.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:33.512+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:33.605 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:33 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:33.512+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:33.893 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:33.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:08:33.893 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:33.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-05-20T23:08:33.894 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:33 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:33.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:08:34.517 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:34.275+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:46.764381+0000 front 2024-05-20T23:07:46.764269+0000 (oldest deadline 2024-05-20T23:08:12.064085+0000) 2024-05-20T23:08:34.518 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:34.275+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.064567+0000 front 2024-05-20T23:07:52.064555+0000 (oldest deadline 2024-05-20T23:08:17.264992+0000) 2024-05-20T23:08:34.518 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6[87423]: 2024-05-20T23:08:34.275+0000 7f82b6ab6700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:00.666526+0000 front 2024-05-20T23:08:00.666642+0000 (oldest deadline 2024-05-20T23:08:25.366363+0000) 2024-05-20T23:08:34.854 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:34.526+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:34.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:34.526+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:34.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:34.526+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:34.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:34 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:34.526+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:35.753 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:35 smithi138 podman[101432]: 2024-05-20 23:08:35.496479103 +0000 UTC m=+5.492859026 container died ceee3eb06ecbcf3d7d5280949f569f4420df0bbba1853491d259afd598d249be (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0) 2024-05-20T23:08:35.753 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:35.502+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:35.753 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:35.502+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:35.753 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:35.502+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:35.753 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:35 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:35.502+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:36.105 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:35 smithi138 podman[101432]: 2024-05-20 23:08:35.861150355 +0000 UTC m=+5.857530272 container cleanup ceee3eb06ecbcf3d7d5280949f569f4420df0bbba1853491d259afd598d249be (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:08:36.105 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:35 smithi138 bash[101432]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6 2024-05-20T23:08:36.477 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:36 smithi138 podman[101449]: 2024-05-20 23:08:36.1208199 +0000 UTC m=+0.625270348 container remove ceee3eb06ecbcf3d7d5280949f569f4420df0bbba1853491d259afd598d249be (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD) 2024-05-20T23:08:36.748 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:36.536+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:36.749 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:36.536+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:36.749 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:36.536+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:36.749 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:36 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:36.536+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:36.749 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:36 smithi138 podman[101534]: 2024-05-20 23:08:36.389884262 +0000 UTC m=+0.017581363 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:08:36.749 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:36 smithi138 podman[101534]: 2024-05-20 23:08:36.488566597 +0000 UTC m=+0.116263720 container create 74d7cc5c2e9d5b645bf9ca3e73338bd08f7250842af6f4b1315a3e1f48a7bb89 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T23:08:36.749 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:36 smithi138 podman[101534]: 2024-05-20 23:08:36.699910049 +0000 UTC m=+0.327607140 container init 74d7cc5c2e9d5b645bf9ca3e73338bd08f7250842af6f4b1315a3e1f48a7bb89 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-05-20T23:08:36.749 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:36 smithi138 podman[101534]: 2024-05-20 23:08:36.703366913 +0000 UTC m=+0.331064015 container start 74d7cc5c2e9d5b645bf9ca3e73338bd08f7250842af6f4b1315a3e1f48a7bb89 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-05-20T23:08:36.749 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:36 smithi138 podman[101534]: 2024-05-20 23:08:36.758304694 +0000 UTC m=+0.386001808 container attach 74d7cc5c2e9d5b645bf9ca3e73338bd08f7250842af6f4b1315a3e1f48a7bb89 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, ceph=True) 2024-05-20T23:08:37.105 INFO:journalctl@ceph.osd.6.smithi138.stdout:May 20 23:08:36 smithi138 podman[101534]: 2024-05-20 23:08:36.98857552 +0000 UTC m=+0.616272624 container died 74d7cc5c2e9d5b645bf9ca3e73338bd08f7250842af6f4b1315a3e1f48a7bb89 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-20T23:08:37.533 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.6.service' 2024-05-20T23:08:37.571 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:08:37.572 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-05-20T23:08:37.572 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-05-20T23:08:37.572 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.7 2024-05-20T23:08:37.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:37.543+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:37.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:37.543+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:37.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:37.543+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:37.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:37.543+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:37.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:37 smithi138 systemd[1]: Stopping Ceph osd.7 for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:08:38.354 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:37.923+0000 7fedcc77a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T23:08:38.355 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:37.923+0000 7fedcc77a700 -1 osd.7 161 *** Got signal Terminated *** 2024-05-20T23:08:38.355 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:37 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:37.923+0000 7fedcc77a700 -1 osd.7 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T23:08:38.854 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:38.512+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:38.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:38.512+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:38.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:38.512+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:38.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:38 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:38.512+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:39.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:39 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:39.519+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:39.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:39 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:39.519+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:39.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:39 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:39.519+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:39.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:39 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:39.519+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:40.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:40 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:40.496+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:40.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:40 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:40.496+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:40.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:40 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:40.496+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:40.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:40 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:40.496+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:41.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:41.530+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:41.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:41.530+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:41.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:41.530+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:41.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:41 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:41.530+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:42.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:42 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:42.543+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2024-05-20T23:07:45.095318+0000 front 2024-05-20T23:07:45.095190+0000 (oldest deadline 2024-05-20T23:08:10.395041+0000) 2024-05-20T23:08:42.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:42 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:42.543+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2024-05-20T23:07:52.696139+0000 front 2024-05-20T23:07:52.696021+0000 (oldest deadline 2024-05-20T23:08:17.395930+0000) 2024-05-20T23:08:42.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:42 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:42.543+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2024-05-20T23:08:01.496973+0000 front 2024-05-20T23:08:01.496997+0000 (oldest deadline 2024-05-20T23:08:27.396897+0000) 2024-05-20T23:08:42.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:42 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:42.543+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2024-05-20T23:08:10.898058+0000 front 2024-05-20T23:08:10.898119+0000 (oldest deadline 2024-05-20T23:08:33.198067+0000) 2024-05-20T23:08:42.855 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:42 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7[91926]: 2024-05-20T23:08:42.543+0000 7fedc8d6e700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.138:6804 osd.4 since back 2024-05-20T23:08:18.899728+0000 front 2024-05-20T23:08:18.899745+0000 (oldest deadline 2024-05-20T23:08:42.399728+0000) 2024-05-20T23:08:43.809 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:43 smithi138 podman[101640]: 2024-05-20 23:08:43.411559237 +0000 UTC m=+5.609430767 container died e41094f18f45c74dcd30f981b34f62923703e0cf01fc70d8e2670a0bc7da8255 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD) 2024-05-20T23:08:43.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:43.555Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=9 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:08:43.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:43.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-05-20T23:08:43.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:43 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:43.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-05-20T23:08:44.104 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:43 smithi138 podman[101640]: 2024-05-20 23:08:43.818026434 +0000 UTC m=+6.015897983 container cleanup e41094f18f45c74dcd30f981b34f62923703e0cf01fc70d8e2670a0bc7da8255 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-20T23:08:44.105 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:43 smithi138 bash[101640]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7 2024-05-20T23:08:44.434 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:44 smithi138 podman[101660]: 2024-05-20 23:08:44.127890208 +0000 UTC m=+0.709845018 container remove e41094f18f45c74dcd30f981b34f62923703e0cf01fc70d8e2670a0bc7da8255 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-05-20T23:08:44.746 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:44 smithi138 podman[101752]: 2024-05-20 23:08:44.34688848 +0000 UTC m=+0.017347217 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T23:08:44.747 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:44 smithi138 podman[101752]: 2024-05-20 23:08:44.453861263 +0000 UTC m=+0.124319982 container create 9710800ec0491e533f2fcf436611519b730d6968928ab715d13f779663136aaf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-05-20T23:08:44.747 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:44 smithi138 podman[101752]: 2024-05-20 23:08:44.681600933 +0000 UTC m=+0.352059668 container init 9710800ec0491e533f2fcf436611519b730d6968928ab715d13f779663136aaf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-05-20T23:08:44.747 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:44 smithi138 podman[101752]: 2024-05-20 23:08:44.685055382 +0000 UTC m=+0.355514118 container start 9710800ec0491e533f2fcf436611519b730d6968928ab715d13f779663136aaf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_CLEAN=True) 2024-05-20T23:08:44.747 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:44 smithi138 podman[101752]: 2024-05-20 23:08:44.756573639 +0000 UTC m=+0.427032367 container attach 9710800ec0491e533f2fcf436611519b730d6968928ab715d13f779663136aaf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-05-20T23:08:45.104 INFO:journalctl@ceph.osd.7.smithi138.stdout:May 20 23:08:44 smithi138 podman[101752]: 2024-05-20 23:08:44.974114404 +0000 UTC m=+0.644573133 container died 9710800ec0491e533f2fcf436611519b730d6968928ab715d13f779663136aaf (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-05-20T23:08:45.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@osd.7.service' 2024-05-20T23:08:45.673 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:08:45.673 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-05-20T23:08:45.673 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-05-20T23:08:45.674 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a 2024-05-20T23:08:46.013 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:45 smithi138 systemd[1]: Stopping Ceph prometheus.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.022Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.022Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.022Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.022Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.022Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.022Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.022Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.022Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.022Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-05-20T23:08:46.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.023Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-05-20T23:08:46.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.023Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-05-20T23:08:46.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a[64005]: ts=2024-05-20T23:08:46.023Z caller=main.go:1147 level=info msg="See you next time!" 2024-05-20T23:08:46.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 podman[101857]: 2024-05-20 23:08:46.032857105 +0000 UTC m=+0.112938527 container died c7fec3619ff9ac2352dd7bf8ed4514bdafd4c6230096af8e58452b87212e3646 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T23:08:46.605 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 podman[101857]: 2024-05-20 23:08:46.489641462 +0000 UTC m=+0.569722882 container cleanup c7fec3619ff9ac2352dd7bf8ed4514bdafd4c6230096af8e58452b87212e3646 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-20T23:08:46.605 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:May 20 23:08:46 smithi138 bash[101857]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-prometheus-a 2024-05-20T23:08:46.906 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@prometheus.a.service' 2024-05-20T23:08:46.939 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T23:08:46.939 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-05-20T23:08:46.939 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 --force --keep-logs 2024-05-20T23:08:48.766 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 23:08:48 smithi055 systemd[1]: Stopping Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:08:49.031 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 23:08:48 smithi055 podman[131242]: 2024-05-20 23:08:48.775556317 +0000 UTC m=+0.106942463 container died b04e9536a7e1687577bb2b38a74a6299f2dcddd8a5fa64e163d2d0b9632680fd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T23:08:49.373 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 23:08:49 smithi055 podman[131242]: 2024-05-20 23:08:49.123431346 +0000 UTC m=+0.454817491 container cleanup b04e9536a7e1687577bb2b38a74a6299f2dcddd8a5fa64e163d2d0b9632680fd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T23:08:49.373 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 23:08:49 smithi055 bash[131242]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a 2024-05-20T23:08:49.732 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 23:08:49 smithi055 podman[131257]: 2024-05-20 23:08:49.38283485 +0000 UTC m=+0.607405824 container remove b04e9536a7e1687577bb2b38a74a6299f2dcddd8a5fa64e163d2d0b9632680fd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-20T23:08:49.732 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 23:08:49 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-20T23:08:49.732 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 23:08:49 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-20T23:08:49.732 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 23:08:49 smithi055 systemd[1]: Stopped Ceph node-exporter.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:08:49.732 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:May 20 23:08:49 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.a.service: Consumed 3.335s CPU time. 2024-05-20T23:08:50.034 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:49 smithi055 systemd[1]: Stopping Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:08:50.035 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:50 smithi055 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a[79687]: ts=2024-05-20T23:08:50.002Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-05-20T23:08:50.319 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:50 smithi055 podman[131364]: 2024-05-20 23:08:50.043986437 +0000 UTC m=+0.138080129 container died df8a6bb3b4f5bc5c892efc00497a4fd42385d42d5051ee43ded926a97e61ff4d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T23:08:50.694 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:50 smithi055 podman[131364]: 2024-05-20 23:08:50.427733903 +0000 UTC m=+0.521827594 container cleanup df8a6bb3b4f5bc5c892efc00497a4fd42385d42d5051ee43ded926a97e61ff4d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T23:08:50.694 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:50 smithi055 bash[131364]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a 2024-05-20T23:08:50.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:50 smithi055 podman[131379]: 2024-05-20 23:08:50.703865952 +0000 UTC m=+0.648496809 container remove df8a6bb3b4f5bc5c892efc00497a4fd42385d42d5051ee43ded926a97e61ff4d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-20T23:08:50.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:50 smithi055 podman[131379]: 2024-05-20 23:08:50.796324981 +0000 UTC m=+0.740955816 volume remove 3bc418356b6b438fa59fc9c59799e75990924a293e951670771e62ab6154cb0f 2024-05-20T23:08:50.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:50 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-20T23:08:50.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:50 smithi055 systemd[1]: Stopped Ceph alertmanager.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:08:50.982 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:May 20 23:08:50 smithi055 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@alertmanager.a.service: Consumed 1.720s CPU time. 2024-05-20T23:09:13.937 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 --force --keep-logs 2024-05-20T23:09:15.676 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 23:09:15 smithi138 systemd[1]: Stopping Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:09:15.966 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 23:09:15 smithi138 podman[102263]: 2024-05-20 23:09:15.685702247 +0000 UTC m=+0.117273062 container died f2df46ba361b007dfc7e0892983f293103c63323e400744de30c198e4c1491e1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T23:09:16.354 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 23:09:16 smithi138 podman[102263]: 2024-05-20 23:09:16.134068552 +0000 UTC m=+0.565639443 container cleanup f2df46ba361b007dfc7e0892983f293103c63323e400744de30c198e4c1491e1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T23:09:16.355 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 23:09:16 smithi138 bash[102263]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b 2024-05-20T23:09:16.716 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 23:09:16 smithi138 podman[102275]: 2024-05-20 23:09:16.393811982 +0000 UTC m=+0.707975855 container remove f2df46ba361b007dfc7e0892983f293103c63323e400744de30c198e4c1491e1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-20T23:09:16.716 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 23:09:16 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-20T23:09:16.716 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 23:09:16 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-20T23:09:16.717 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 23:09:16 smithi138 systemd[1]: Stopped Ceph node-exporter.b for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:09:16.717 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:May 20 23:09:16 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@node-exporter.b.service: Consumed 3.110s CPU time. 2024-05-20T23:09:16.972 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:16 smithi138 systemd[1]: Stopping Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968... 2024-05-20T23:09:16.973 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=server t=2024-05-20T23:09:16.972501873Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-05-20T23:09:16.973 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=tracing t=2024-05-20T23:09:16.972894691Z level=info msg="Closing tracing" 2024-05-20T23:09:16.973 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:16 smithi138 ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a[66215]: logger=ticker t=2024-05-20T23:09:16.972902855Z level=info msg=stopped last_tick=2024-05-20T23:09:10Z 2024-05-20T23:09:17.295 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:16 smithi138 podman[102384]: 2024-05-20 23:09:16.982660416 +0000 UTC m=+0.115149507 container died c12c9c6bb205769261cf19338518ec15b944fec1674a60fdfcf72de8b3ca1b09 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-20T23:09:17.296 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:17 smithi138 podman[102384]: 2024-05-20 23:09:17.30581218 +0000 UTC m=+0.438301270 container cleanup c12c9c6bb205769261cf19338518ec15b944fec1674a60fdfcf72de8b3ca1b09 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-20T23:09:17.604 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:17 smithi138 bash[102384]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a 2024-05-20T23:09:17.604 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:17 smithi138 podman[102398]: 2024-05-20 23:09:17.507302474 +0000 UTC m=+0.522241688 container remove c12c9c6bb205769261cf19338518ec15b944fec1674a60fdfcf72de8b3ca1b09 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-20T23:09:18.105 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:17 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-20T23:09:18.105 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:17 smithi138 systemd[1]: Stopped Ceph grafana.a for dd0bace8-16f7-11ef-bc9a-c7b262605968. 2024-05-20T23:09:18.105 INFO:journalctl@ceph.grafana.a.smithi138.stdout:May 20 23:09:17 smithi138 systemd[1]: ceph-dd0bace8-16f7-11ef-bc9a-c7b262605968@grafana.a.service: Consumed 11.208s CPU time. 2024-05-20T23:09:29.430 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T23:09:29.465 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T23:09:29.492 INFO:tasks.cephadm:Archiving crash dumps... 2024-05-20T23:09:29.494 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/crash to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802/remote/smithi055/crash 2024-05-20T23:09:29.495 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/crash -- . 2024-05-20T23:09:29.549 INFO:teuthology.orchestra.run.smithi055.stderr:tar: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/crash: Cannot open: No such file or directory 2024-05-20T23:09:29.549 INFO:teuthology.orchestra.run.smithi055.stderr:tar: Error is not recoverable: exiting now 2024-05-20T23:09:29.551 DEBUG:teuthology.misc:Transferring archived files from smithi138:/var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/crash to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802/remote/smithi138/crash 2024-05-20T23:09:29.552 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/crash -- . 2024-05-20T23:09:29.621 INFO:teuthology.orchestra.run.smithi138.stderr:tar: /var/lib/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/crash: Cannot open: No such file or directory 2024-05-20T23:09:29.621 INFO:teuthology.orchestra.run.smithi138.stderr:tar: Error is not recoverable: exiting now 2024-05-20T23:09:29.622 INFO:tasks.cephadm:Checking cluster log for badness... 2024-05-20T23:09:29.623 DEBUG:teuthology.orchestra.run.smithi055:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-20T23:09:29.654 INFO:tasks.cephadm:Compressing logs... 2024-05-20T23:09:29.654 DEBUG:teuthology.orchestra.run.smithi055:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-20T23:09:29.697 DEBUG:teuthology.orchestra.run.smithi138:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-20T23:09:29.721 INFO:teuthology.orchestra.run.smithi138.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.log 2024-05-20T23:09:29.722 INFO:teuthology.orchestra.run.smithi138.stderr:: No such file or directory 2024-05-20T23:09:29.722 INFO:teuthology.orchestra.run.smithi138.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.audit.log 2024-05-20T23:09:29.722 INFO:teuthology.orchestra.run.smithi138.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.7.log 2024-05-20T23:09:29.722 INFO:teuthology.orchestra.run.smithi138.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.log: /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.5.log 2024-05-20T23:09:29.727 INFO:teuthology.orchestra.run.smithi055.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.log 2024-05-20T23:09:29.728 INFO:teuthology.orchestra.run.smithi055.stderr:: No such file or directory 2024-05-20T23:09:29.728 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.2.log 2024-05-20T23:09:29.728 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.audit.log 2024-05-20T23:09:29.728 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.log: /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.3.log 2024-05-20T23:09:29.728 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.1.log 2024-05-20T23:09:29.728 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mgr.y.log 2024-05-20T23:09:29.728 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.3.log: /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.0.log 2024-05-20T23:09:29.728 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mon.c.log 2024-05-20T23:09:29.728 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mon.a.log 2024-05-20T23:09:29.730 INFO:teuthology.orchestra.run.smithi138.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-client.rgw.foo.smithi138.sfpuja.log 2024-05-20T23:09:29.730 INFO:teuthology.orchestra.run.smithi138.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.4.log 2024-05-20T23:09:29.730 INFO:teuthology.orchestra.run.smithi138.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.5.log: /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-client.rgw.foo.smithi138.sfpuja.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mgr.x.log 2024-05-20T23:09:29.730 INFO:teuthology.orchestra.run.smithi138.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.4.log: 75.6%gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.6.log 2024-05-20T23:09:29.730 INFO:teuthology.orchestra.run.smithi138.stderr: -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-client.rgw.foo.smithi138.sfpuja.log.gz 2024-05-20T23:09:29.730 INFO:teuthology.orchestra.run.smithi138.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.cephadm.log 2024-05-20T23:09:29.730 INFO:teuthology.orchestra.run.smithi138.stderr:gzip -5 --verbose --/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.6.log: /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-volume.log 2024-05-20T23:09:29.730 INFO:teuthology.orchestra.run.smithi138.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.cephadm.log: 89.0% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.log.gz 2024-05-20T23:09:29.730 INFO:teuthology.orchestra.run.smithi138.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mon.b.log 2024-05-20T23:09:29.731 INFO:teuthology.orchestra.run.smithi138.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-volume.log: 91.6% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.cephadm.log.gz 2024-05-20T23:09:29.731 INFO:teuthology.orchestra.run.smithi138.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-20T23:09:29.731 INFO:teuthology.orchestra.run.smithi138.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mon.b.log: 91.0% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.audit.log.gz 2024-05-20T23:09:29.731 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mon.c.log: 93.9% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.log.gz 2024-05-20T23:09:29.734 INFO:teuthology.orchestra.run.smithi055.stderr: 94.4% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.audit.log.gz 2024-05-20T23:09:29.735 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.cephadm.log 2024-05-20T23:09:29.735 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/tcmu-runner.log 2024-05-20T23:09:29.739 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.cephadm.log: 94.9% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph.cephadm.log.gz 2024-05-20T23:09:29.741 INFO:teuthology.orchestra.run.smithi138.stderr: 91.0% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-volume.log.gz 2024-05-20T23:09:29.748 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-volume.log 2024-05-20T23:09:29.749 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/tcmu-runner.log: 90.2% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/tcmu-runner.log.gz 2024-05-20T23:09:29.761 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-client.rgw.foo.smithi055.ymnqng.log 2024-05-20T23:09:29.774 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-20T23:09:29.775 INFO:teuthology.orchestra.run.smithi138.stderr:/var/log/ceph/cephadm.log: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-20T23:09:29.775 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-client.rgw.foo.smithi055.ymnqng.log: 76.0% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-client.rgw.foo.smithi055.ymnqng.log.gz 2024-05-20T23:09:29.777 INFO:teuthology.orchestra.run.smithi055.stderr: 91.0% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-volume.log.gz 2024-05-20T23:09:29.800 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/cephadm.log: 92.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-20T23:09:29.878 INFO:teuthology.orchestra.run.smithi138.stderr: 89.7% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mgr.x.log.gz 2024-05-20T23:09:30.094 INFO:teuthology.orchestra.run.smithi055.stderr: 89.0% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mgr.y.log.gz 2024-05-20T23:09:30.504 INFO:teuthology.orchestra.run.smithi055.stderr: 92.5% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mon.c.log.gz 2024-05-20T23:09:30.699 INFO:teuthology.orchestra.run.smithi138.stderr: 92.3% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mon.b.log.gz 2024-05-20T23:09:31.179 INFO:teuthology.orchestra.run.smithi138.stderr: 93.2% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.6.log.gz 2024-05-20T23:09:31.197 INFO:teuthology.orchestra.run.smithi055.stderr: 93.0% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.2.log.gz 2024-05-20T23:09:31.455 INFO:teuthology.orchestra.run.smithi055.stderr: 91.0% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-mon.a.log.gz 2024-05-20T23:09:31.558 INFO:teuthology.orchestra.run.smithi138.stderr: 93.4% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.5.log.gz 2024-05-20T23:09:31.596 INFO:teuthology.orchestra.run.smithi138.stderr: 93.8% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.7.log.gz 2024-05-20T23:09:32.019 INFO:teuthology.orchestra.run.smithi055.stderr: 93.4% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.0.log.gz 2024-05-20T23:09:32.052 INFO:teuthology.orchestra.run.smithi138.stderr: 93.6% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.4.log.gz 2024-05-20T23:09:32.054 INFO:teuthology.orchestra.run.smithi138.stderr: 2024-05-20T23:09:32.054 INFO:teuthology.orchestra.run.smithi138.stderr:real 0m2.343s 2024-05-20T23:09:32.054 INFO:teuthology.orchestra.run.smithi138.stderr:user 0m8.350s 2024-05-20T23:09:32.054 INFO:teuthology.orchestra.run.smithi138.stderr:sys 0m0.347s 2024-05-20T23:09:32.117 INFO:teuthology.orchestra.run.smithi055.stderr: 93.4% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.1.log.gz 2024-05-20T23:09:32.129 INFO:teuthology.orchestra.run.smithi055.stderr: 93.4% -- replaced with /var/log/ceph/dd0bace8-16f7-11ef-bc9a-c7b262605968/ceph-osd.3.log.gz 2024-05-20T23:09:32.131 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-05-20T23:09:32.131 INFO:teuthology.orchestra.run.smithi055.stderr:real 0m2.420s 2024-05-20T23:09:32.131 INFO:teuthology.orchestra.run.smithi055.stderr:user 0m11.116s 2024-05-20T23:09:32.131 INFO:teuthology.orchestra.run.smithi055.stderr:sys 0m0.387s 2024-05-20T23:09:32.131 INFO:tasks.cephadm:Archiving logs... 2024-05-20T23:09:32.131 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/log/ceph to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802/remote/smithi055/log 2024-05-20T23:09:32.133 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-20T23:09:33.633 DEBUG:teuthology.misc:Transferring archived files from smithi138:/var/log/ceph to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802/remote/smithi138/log 2024-05-20T23:09:33.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-20T23:09:34.814 INFO:tasks.cephadm:Removing cluster... 2024-05-20T23:09:34.815 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 --force 2024-05-20T23:09:35.123 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dd0bace8-16f7-11ef-bc9a-c7b262605968 --force 2024-05-20T23:09:35.414 INFO:tasks.cephadm:Removing cephadm ... 2024-05-20T23:09:35.414 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-20T23:09:35.431 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-20T23:09:35.445 INFO:tasks.cephadm:Teardown complete 2024-05-20T23:09:35.446 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-05-20T23:09:35.455 INFO:teuthology.task.clock:Checking final clock skew... 2024-05-20T23:09:35.456 DEBUG:teuthology.orchestra.run.smithi055:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T23:09:35.472 DEBUG:teuthology.orchestra.run.smithi138:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T23:09:35.485 INFO:teuthology.orchestra.run.smithi055.stderr:bash: line 1: ntpq: command not found 2024-05-20T23:09:35.489 INFO:teuthology.orchestra.run.smithi055.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T23:09:35.489 INFO:teuthology.orchestra.run.smithi055.stdout:=============================================================================== 2024-05-20T23:09:35.489 INFO:teuthology.orchestra.run.smithi055.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 +586us[ +655us] +/- 37ms 2024-05-20T23:09:35.489 INFO:teuthology.orchestra.run.smithi055.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 49 -110us[ -110us] +/- 69ms 2024-05-20T23:09:35.489 INFO:teuthology.orchestra.run.smithi055.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 54 -883us[ -814us] +/- 70ms 2024-05-20T23:09:35.489 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T23:09:35.499 INFO:teuthology.orchestra.run.smithi138.stderr:bash: line 1: ntpq: command not found 2024-05-20T23:09:35.503 INFO:teuthology.orchestra.run.smithi138.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T23:09:35.503 INFO:teuthology.orchestra.run.smithi138.stdout:=============================================================================== 2024-05-20T23:09:35.503 INFO:teuthology.orchestra.run.smithi138.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 +591us[ +672us] +/- 37ms 2024-05-20T23:09:35.503 INFO:teuthology.orchestra.run.smithi138.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 52 -105us[ -105us] +/- 69ms 2024-05-20T23:09:35.503 INFO:teuthology.orchestra.run.smithi138.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 54 -882us[ -800us] +/- 70ms 2024-05-20T23:09:35.503 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T23:09:35.504 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-05-20T23:09:35.513 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-05-20T23:09:35.513 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-05-20T23:09:35.527 DEBUG:teuthology.orchestra.run.smithi055:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-20T23:09:35.621 DEBUG:teuthology.orchestra.run.smithi138:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-20T23:09:35.697 DEBUG:teuthology.orchestra.run.smithi055:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-20T23:09:35.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T23:09:35.727 DEBUG:teuthology.orchestra.run.smithi138:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-20T23:09:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T23:09:35.755 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-05-20T23:09:35.765 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-05-20T23:09:35.779 INFO:teuthology.task.internal:Duration was 2912.239044 seconds 2024-05-20T23:09:35.779 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-05-20T23:09:35.793 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-05-20T23:09:35.794 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-20T23:09:35.796 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-20T23:09:35.834 INFO:teuthology.orchestra.run.smithi055.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T23:09:35.835 INFO:teuthology.orchestra.run.smithi138.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T23:09:36.268 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-05-20T23:09:36.268 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi055.front.sepia.ceph.com 2024-05-20T23:09:36.268 DEBUG:teuthology.orchestra.run.smithi055:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-20T23:09:36.332 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi138.front.sepia.ceph.com 2024-05-20T23:09:36.332 DEBUG:teuthology.orchestra.run.smithi138:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-20T23:09:36.365 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-05-20T23:09:36.366 DEBUG:teuthology.orchestra.run.smithi055:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-20T23:09:36.375 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-20T23:09:36.436 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-05-20T23:09:36.436 DEBUG:teuthology.orchestra.run.smithi055:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-20T23:09:36.444 DEBUG:teuthology.orchestra.run.smithi138:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-20T23:09:37.052 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-05-20T23:09:37.062 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-05-20T23:09:37.062 DEBUG:teuthology.orchestra.run.smithi055:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-20T23:09:37.087 DEBUG:teuthology.orchestra.run.smithi138:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-20T23:09:37.113 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-05-20T23:09:37.122 DEBUG:teuthology.orchestra.run.smithi055:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-20T23:09:37.130 DEBUG:teuthology.orchestra.run.smithi138:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-20T23:09:37.151 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern = core 2024-05-20T23:09:37.176 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = core 2024-05-20T23:09:37.198 DEBUG:teuthology.orchestra.run.smithi055:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-20T23:09:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T23:09:37.225 DEBUG:teuthology.orchestra.run.smithi138:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-20T23:09:37.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T23:09:37.253 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-05-20T23:09:37.263 INFO:teuthology.task.internal:Transferring archived files... 2024-05-20T23:09:37.263 DEBUG:teuthology.misc:Transferring archived files from smithi055:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802/remote/smithi055 2024-05-20T23:09:37.263 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-20T23:09:37.315 DEBUG:teuthology.misc:Transferring archived files from smithi138:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715802/remote/smithi138 2024-05-20T23:09:37.316 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-20T23:09:37.355 INFO:teuthology.task.internal:Removing archive directory... 2024-05-20T23:09:37.355 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-20T23:09:37.357 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-20T23:09:37.402 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-05-20T23:09:37.414 INFO:teuthology.task.internal:Not uploading archives. 2024-05-20T23:09:37.414 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-05-20T23:09:37.425 INFO:teuthology.task.internal:Tidying up after the test... 2024-05-20T23:09:37.425 DEBUG:teuthology.orchestra.run.smithi055:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-20T23:09:37.427 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-20T23:09:37.440 INFO:teuthology.orchestra.run.smithi055.stdout: 265296 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 20 23:09 /home/ubuntu/cephtest 2024-05-20T23:09:37.457 INFO:teuthology.orchestra.run.smithi138.stdout: 265296 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 20 23:09 /home/ubuntu/cephtest 2024-05-20T23:09:37.458 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-05-20T23:09:37.468 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-05-20T23:09:37.493 INFO:teuthology.run:Summary data: description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2912.2390444278717 owner: scheduled_adking@teuthology success: true 2024-05-20T23:09:37.494 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-20T23:09:37.603 INFO:teuthology.run:pass